diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1040.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1040.log new file mode 100644 index 0000000000000000000000000000000000000000..d4f0dd5b19870030926a76721afe62e405c18b78 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1040.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_daemon_1040 ++ mkdir -p /tmp/mcore2hf_daemon_1040 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 1040 ']' ++ MANUAL_ITER=1040 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040 ++ CHECKPOINT_BASENAME=load_view1040 ++ [[ load_view1040 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_daemon_1040/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_daemon_1040/mcore2hf-config.k3xZc6 ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_daemon_1040/mcore2hf-config.k3xZc6:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_daemon_1040/mcore2hf-config.k3xZc6:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040 +W1014 07:21:44.510000 15798 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1014 07:21:44.510000 15798 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=524288 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=524288 +++ MAX_POSITION_EMBEDDINGS=524288 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040 +W1014 07:22:15.232000 16880 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1014 07:22:15.232000 16880 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 524288 ']' ++ '[' 524288 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_daemon_1040/mcore2hf-config.k3xZc6 +W1014 07:22:28.437000 17318 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1014 07:22:28.437000 17318 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_daemon_1040/mcore2hf-config.k3xZc6/config.json' +✅config.json: /tmp/mcore2hf_daemon_1040/mcore2hf-config.k3xZc6/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_daemon_1040/mcore2hf-config.k3xZc6/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 524288, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_daemon_1040/mcore2hf-config.k3xZc6 ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040 1040 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040 ++ local ITER_NUM=1040 ++ '[' -z 1040 ']' +++ printf %07d 1040 ++ local ITER_BASENAME=iter_0001040 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040/iter_0001040 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040/iter_0001040 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040/latest_checkpointed_iteration.txt ++ LATEST_VALUE=1040 ++ '[' 1040 = 1040 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=1040 ++ echo -e '\033[32mLatest checkpoint iteration: 1040\033[0m\n' +Latest checkpoint iteration: 1040 + ++ I=1040 ++ [[ 1040 =~ ^[0-9]+$ ]] +++ printf %07d 1040 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040/iter_0001040 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040/iter_0001040/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040/iter_1040-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1040-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1040-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1040-hf true false bf16 /tmp/mcore2hf_daemon_1040/mcore2hf-config.k3xZc6 1040 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1040-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 47487 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_daemon_1040/mcore2hf-config.k3xZc6 --hf-dir /tmp/mcore2hf_daemon_1040/mcore2hf-config.k3xZc6 --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 524288 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1040-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 1040 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1014 07:22:48.486000 17937 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1014 07:22:48.486000 17937 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 1040 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_daemon_1040/mcore2hf-config.k3xZc6 + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 524288 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1040-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_daemon_1040/mcore2hf-config.k3xZc6 + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.127 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.004 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_k3xZc6/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_k3xZc6/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_k3xZc6.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040 at iteration 1040 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1040 [ t 1/1, p 1/1 ] at iteration 1040 +[convert] synchronizer created in 380.92s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0771484375 mean=-0.000499725341796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.1025390625 mean=0.001617431640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0791015625 mean=-0.00011968612670898438 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10595703125 mean=-4.7206878662109375e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0849609375 mean=-0.00021457672119140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.080078125 mean=0.000820159912109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08837890625 mean=-0.0002288818359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.078125 mean=0.00144195556640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09619140625 mean=-0.0002231597900390625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07421875 mean=-0.00077056884765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09130859375 mean=-7.867813110351562e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06298828125 mean=-0.0001888275146484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09375 mean=-8.487701416015625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06298828125 mean=0.00016880035400390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08837890625 mean=4.172325134277344e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.056640625 mean=-0.0004673004150390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08447265625 mean=5.9604644775390625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.054931640625 mean=-0.000537872314453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08251953125 mean=2.3603439331054688e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06640625 mean=-0.00041961669921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.00019550323486328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0546875 mean=-0.0006103515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07177734375 mean=0.0001087188720703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04345703125 mean=-0.0010528564453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.00014781951904296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.046875 mean=-0.0010223388671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07373046875 mean=0.00022029876708984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.049560546875 mean=-0.0006256103515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.072265625 mean=0.0002918243408203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0478515625 mean=-0.000579833984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06982421875 mean=0.0003147125244140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.046630859375 mean=-0.00141143798828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06591796875 mean=0.0002651214599609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04736328125 mean=-0.0021209716796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06884765625 mean=0.000553131103515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.049560546875 mean=-0.00173187255859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06787109375 mean=0.00034332275390625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05078125 mean=-0.00121307373046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060546875 mean=0.0003681182861328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.041748046875 mean=-0.00213623046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0556640625 mean=0.000293731689453125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0439453125 mean=-0.003021240234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0537109375 mean=0.0004291534423828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03759765625 mean=-0.002166748046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05224609375 mean=0.0004253387451171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03564453125 mean=-0.002410888671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049560546875 mean=0.0003795623779296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.033935546875 mean=-0.00160980224609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=0.00030517578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.031494140625 mean=-0.00164794921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.051513671875 mean=0.0004138946533203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0306396484375 mean=-0.0018463134765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.047607421875 mean=0.0003757476806640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0272216796875 mean=-0.000972747802734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04833984375 mean=0.000518798828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0242919921875 mean=-0.00079345703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.051513671875 mean=0.000812530517578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0255126953125 mean=-0.000812530517578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048828125 mean=0.00052642822265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0264892578125 mean=-0.000400543212890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.046142578125 mean=0.00012969970703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.025390625 mean=-0.0004405975341796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=8.535385131835938e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02783203125 mean=-6.818771362304688e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.000152587890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02685546875 mean=5.53131103515625e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05712890625 mean=4.291534423828125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.032470703125 mean=0.0003414154052734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05908203125 mean=4.7206878662109375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043212890625 mean=0.00096893310546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.058837890625 mean=9.822845458984375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04296875 mean=0.0013275146484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0576171875 mean=0.0001068115234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037109375 mean=0.00145721435546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.055908203125 mean=6.246566772460938e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.036865234375 mean=0.00122833251953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052490234375 mean=0.00010251998901367188 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02587890625 mean=0.000316619873046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=9.393692016601562e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0242919921875 mean=0.000396728515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0439453125 mean=0.0003185272216796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0211181640625 mean=-0.0004863739013671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=0.0001659393310546875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.020263671875 mean=-0.00015544891357421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03564453125 mean=0.00042724609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0218505859375 mean=-0.0003528594970703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.035888671875 mean=8.440017700195312e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02392578125 mean=-0.00087738037109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0361328125 mean=0.0002956390380859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02099609375 mean=-0.000629425048828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036376953125 mean=0.000179290771484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02392578125 mean=-0.00083160400390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03369140625 mean=-9.298324584960938e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0198974609375 mean=-0.0004634857177734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.033935546875 mean=-0.00012302398681640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0216064453125 mean=-0.00110626220703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=-0.00010633468627929688 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0240478515625 mean=-0.000743865966796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03955078125 mean=1.633167266845703e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0242919921875 mean=-0.00102996826171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.040283203125 mean=0.00020122528076171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.031005859375 mean=-0.00131988525390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037353515625 mean=0.00011873245239257812 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05322265625 mean=-0.0024261474609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=9.107589721679688e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04150390625 mean=-0.0019073486328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04296875 mean=-0.0001316070556640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.044189453125 mean=0.0026092529296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.95s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1040-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 90.91s +Conversion finished in 474.1598627567291 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1040-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1040-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_daemon_1040/mcore2hf-config.k3xZc6 ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1120.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1120.log new file mode 100644 index 0000000000000000000000000000000000000000..763dd0d0650370f7101616dee62b2b098e1ff5cf --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1120.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_daemon_1120 ++ mkdir -p /tmp/mcore2hf_daemon_1120 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 1120 ']' ++ MANUAL_ITER=1120 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120 ++ CHECKPOINT_BASENAME=load_view1120 ++ [[ load_view1120 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_daemon_1120/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_daemon_1120/mcore2hf-config.P1yGkA ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_daemon_1120/mcore2hf-config.P1yGkA:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_daemon_1120/mcore2hf-config.P1yGkA:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120 +W1014 12:23:40.317000 378760 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1014 12:23:40.317000 378760 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=524288 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=524288 +++ MAX_POSITION_EMBEDDINGS=524288 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120 +W1014 12:23:55.280000 379361 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1014 12:23:55.280000 379361 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 524288 ']' ++ '[' 524288 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_daemon_1120/mcore2hf-config.P1yGkA +W1014 12:24:08.386000 379866 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1014 12:24:08.386000 379866 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_daemon_1120/mcore2hf-config.P1yGkA/config.json' +✅config.json: /tmp/mcore2hf_daemon_1120/mcore2hf-config.P1yGkA/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_daemon_1120/mcore2hf-config.P1yGkA/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 524288, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_daemon_1120/mcore2hf-config.P1yGkA ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120 1120 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120 ++ local ITER_NUM=1120 ++ '[' -z 1120 ']' +++ printf %07d 1120 ++ local ITER_BASENAME=iter_0001120 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120/iter_0001120 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120/iter_0001120 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120/latest_checkpointed_iteration.txt ++ LATEST_VALUE=1120 ++ '[' 1120 = 1120 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=1120 ++ echo -e '\033[32mLatest checkpoint iteration: 1120\033[0m\n' +Latest checkpoint iteration: 1120 + ++ I=1120 ++ [[ 1120 =~ ^[0-9]+$ ]] +++ printf %07d 1120 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120/iter_0001120 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120/iter_0001120/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120/iter_1120-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1120-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1120-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1120-hf true false bf16 /tmp/mcore2hf_daemon_1120/mcore2hf-config.P1yGkA 1120 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1120-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 44135 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_daemon_1120/mcore2hf-config.P1yGkA --hf-dir /tmp/mcore2hf_daemon_1120/mcore2hf-config.P1yGkA --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 524288 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1120-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 1120 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1014 12:24:28.505000 380449 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1014 12:24:28.505000 380449 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 1120 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_daemon_1120/mcore2hf-config.P1yGkA + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 524288 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1120-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_daemon_1120/mcore2hf-config.P1yGkA + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.137 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_P1yGkA/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_P1yGkA/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_P1yGkA.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120 at iteration 1120 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1120 [ t 1/1, p 1/1 ] at iteration 1120 +[convert] synchronizer created in 388.31s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07666015625 mean=-0.000499725341796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.1025390625 mean=0.001556396484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0791015625 mean=-0.00012111663818359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10546875 mean=4.76837158203125e-06 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0849609375 mean=-0.000217437744140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.080078125 mean=0.000843048095703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08837890625 mean=-0.00023746490478515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07763671875 mean=0.001495361328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09619140625 mean=-0.00022220611572265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07421875 mean=-0.000835418701171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09130859375 mean=-8.20159912109375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0625 mean=-0.0001239776611328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09375 mean=-8.535385131835938e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06298828125 mean=9.298324584960938e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.087890625 mean=4.029273986816406e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.056640625 mean=-0.0004558563232421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.083984375 mean=6.246566772460938e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0546875 mean=-0.000408172607421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08251953125 mean=1.5854835510253906e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06591796875 mean=-0.0002689361572265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.00019931793212890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.054443359375 mean=-0.00041961669921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07177734375 mean=0.00010395050048828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04345703125 mean=-0.00096893310546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.000148773193359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.046630859375 mean=-0.000957489013671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07373046875 mean=0.00022220611572265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04931640625 mean=-0.000492095947265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.072265625 mean=0.0002880096435546875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.047607421875 mean=-0.00046539306640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06982421875 mean=0.0003185272216796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.046630859375 mean=-0.00148773193359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06591796875 mean=0.0002689361572265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.046875 mean=-0.001953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06884765625 mean=0.000545501708984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04931640625 mean=-0.00173187255859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06787109375 mean=0.000335693359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.050537109375 mean=-0.00119781494140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060546875 mean=0.0003662109375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.041748046875 mean=-0.0021514892578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0556640625 mean=0.0002899169921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043701171875 mean=-0.0028839111328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0537109375 mean=0.0004329681396484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037353515625 mean=-0.0021514892578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052490234375 mean=0.000423431396484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.035400390625 mean=-0.0023651123046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049560546875 mean=0.0003833770751953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03369140625 mean=-0.00159454345703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=0.00030517578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03125 mean=-0.00152587890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.051513671875 mean=0.0004138946533203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0302734375 mean=-0.0017547607421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.047607421875 mean=0.00037384033203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.027099609375 mean=-0.000946044921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04833984375 mean=0.00052642822265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0244140625 mean=-0.000732421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.051513671875 mean=0.000812530517578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0255126953125 mean=-0.000751495361328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048828125 mean=0.000530242919921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0262451171875 mean=-0.00040435791015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.046142578125 mean=0.0001316070556640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0255126953125 mean=-0.0003509521484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=8.58306884765625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.027587890625 mean=8.726119995117188e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.00014972686767578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0269775390625 mean=5.745887756347656e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05712890625 mean=4.220008850097656e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0322265625 mean=0.000255584716796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05908203125 mean=4.76837158203125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04296875 mean=0.00104522705078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.058837890625 mean=9.632110595703125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04296875 mean=0.00128173828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0576171875 mean=0.00010824203491210938 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037109375 mean=0.0015106201171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.055908203125 mean=6.4849853515625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037109375 mean=0.0012969970703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052490234375 mean=0.00010347366333007812 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02587890625 mean=0.0003376007080078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=9.489059448242188e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.024169921875 mean=0.000316619873046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0439453125 mean=0.0003185272216796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02099609375 mean=-0.0004215240478515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=0.0001678466796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0201416015625 mean=-0.00014209747314453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03564453125 mean=0.000438690185546875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.021728515625 mean=-0.00034332275390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.035888671875 mean=8.916854858398438e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.023681640625 mean=-0.0008544921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0361328125 mean=0.000301361083984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02099609375 mean=-0.000560760498046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036376953125 mean=0.0001811981201171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0238037109375 mean=-0.000820159912109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03369140625 mean=-9.107589721679688e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.019775390625 mean=-0.000514984130859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.033935546875 mean=-0.00012302398681640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.021484375 mean=-0.00113677978515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0380859375 mean=-0.00010347366333007812 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0240478515625 mean=-0.0007781982421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03955078125 mean=1.3053417205810547e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.024169921875 mean=-0.0011444091796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.040283203125 mean=0.0002002716064453125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.031005859375 mean=-0.00136566162109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037353515625 mean=0.00011730194091796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05322265625 mean=-0.002349853515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=8.440017700195312e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.041259765625 mean=-0.0019378662109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04296875 mean=-0.00012969970703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.044189453125 mean=0.002685546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.94s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1120-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 75.89s +Conversion finished in 466.6087989807129 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1120-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1120-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_daemon_1120/mcore2hf-config.P1yGkA ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1200.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1200.log new file mode 100644 index 0000000000000000000000000000000000000000..39620373cd9a2f689949838e3ed1408ffb8cb26b --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1200.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_daemon_1200 ++ mkdir -p /tmp/mcore2hf_daemon_1200 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 1200 ']' ++ MANUAL_ITER=1200 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200 ++ CHECKPOINT_BASENAME=load_view1200 ++ [[ load_view1200 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_daemon_1200/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_daemon_1200/mcore2hf-config.Ah9Wjg ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_daemon_1200/mcore2hf-config.Ah9Wjg:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_daemon_1200/mcore2hf-config.Ah9Wjg:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200 +W1014 17:25:25.447000 774956 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1014 17:25:25.447000 774956 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=524288 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=524288 +++ MAX_POSITION_EMBEDDINGS=524288 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200 +W1014 17:25:40.427000 775589 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1014 17:25:40.427000 775589 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 524288 ']' ++ '[' 524288 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_daemon_1200/mcore2hf-config.Ah9Wjg +W1014 17:25:53.520000 776017 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1014 17:25:53.520000 776017 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_daemon_1200/mcore2hf-config.Ah9Wjg/config.json' +✅config.json: /tmp/mcore2hf_daemon_1200/mcore2hf-config.Ah9Wjg/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_daemon_1200/mcore2hf-config.Ah9Wjg/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 524288, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_daemon_1200/mcore2hf-config.Ah9Wjg ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200 1200 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200 ++ local ITER_NUM=1200 ++ '[' -z 1200 ']' +++ printf %07d 1200 ++ local ITER_BASENAME=iter_0001200 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200/iter_0001200 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200/iter_0001200 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200/latest_checkpointed_iteration.txt ++ LATEST_VALUE=1200 ++ '[' 1200 = 1200 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=1200 ++ echo -e '\033[32mLatest checkpoint iteration: 1200\033[0m\n' +Latest checkpoint iteration: 1200 + ++ I=1200 ++ [[ 1200 =~ ^[0-9]+$ ]] +++ printf %07d 1200 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200/iter_0001200 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200/iter_0001200/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200/iter_1200-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1200-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1200-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1200-hf true false bf16 /tmp/mcore2hf_daemon_1200/mcore2hf-config.Ah9Wjg 1200 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1200-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 36549 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_daemon_1200/mcore2hf-config.Ah9Wjg --hf-dir /tmp/mcore2hf_daemon_1200/mcore2hf-config.Ah9Wjg --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 524288 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1200-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 1200 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1014 17:26:14.108000 776628 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1014 17:26:14.108000 776628 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 1200 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_daemon_1200/mcore2hf-config.Ah9Wjg + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 524288 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1200-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_daemon_1200/mcore2hf-config.Ah9Wjg + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.204 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_Ah9Wjg/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_Ah9Wjg/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_Ah9Wjg.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200 at iteration 1200 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1200 [ t 1/1, p 1/1 ] at iteration 1200 +[convert] synchronizer created in 397.20s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07666015625 mean=-0.00049591064453125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.1025390625 mean=0.001434326171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0791015625 mean=-0.00013637542724609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10546875 mean=-5.91278076171875e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0849609375 mean=-0.00022602081298828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.080078125 mean=0.000843048095703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08837890625 mean=-0.0002613067626953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07763671875 mean=0.0014190673828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09619140625 mean=-0.00022220611572265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07373046875 mean=-0.0008697509765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09130859375 mean=-8.869171142578125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.062255859375 mean=-3.743171691894531e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09375 mean=-9.012222290039062e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0625 mean=0.00015735626220703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.087890625 mean=3.337860107421875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.056396484375 mean=-0.000484466552734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.083984375 mean=6.079673767089844e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0546875 mean=-0.0004825592041015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08251953125 mean=1.0609626770019531e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06591796875 mean=-0.000247955322265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.0001964569091796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.054443359375 mean=-0.000431060791015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07177734375 mean=9.822845458984375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04296875 mean=-0.00101470947265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.00014781951904296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.046630859375 mean=-0.0010528564453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07373046875 mean=0.00022029876708984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.049072265625 mean=-0.000507354736328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.072265625 mean=0.0002899169921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04736328125 mean=-0.0004482269287109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06982421875 mean=0.0003223419189453125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.046142578125 mean=-0.0015869140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06591796875 mean=0.0002651214599609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.046630859375 mean=-0.001922607421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06884765625 mean=0.000545501708984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04931640625 mean=-0.00177001953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06787109375 mean=0.000335693359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.050537109375 mean=-0.0012664794921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060546875 mean=0.0003643035888671875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.041259765625 mean=-0.0022430419921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0556640625 mean=0.0002803802490234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043701171875 mean=-0.0030364990234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.053466796875 mean=0.00042724609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037109375 mean=-0.002166748046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05224609375 mean=0.000423431396484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03515625 mean=-0.00225830078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049560546875 mean=0.0003833770751953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.033447265625 mean=-0.0017242431640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=0.00030517578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03125 mean=-0.001617431640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.051513671875 mean=0.0004100799560546875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0301513671875 mean=-0.00165557861328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.047607421875 mean=0.0003681182861328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0269775390625 mean=-0.000881195068359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04833984375 mean=0.000518798828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0242919921875 mean=-0.00074005126953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.051513671875 mean=0.0008087158203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0252685546875 mean=-0.000759124755859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048828125 mean=0.000530242919921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.026123046875 mean=-0.0003662109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.046142578125 mean=0.000133514404296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0252685546875 mean=-0.0003108978271484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=7.867813110351562e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0274658203125 mean=-1.2218952178955078e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.00014209747314453125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02685546875 mean=6.580352783203125e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05712890625 mean=4.172325134277344e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.032470703125 mean=0.00041961669921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05908203125 mean=4.410743713378906e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04296875 mean=0.00107574462890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.058837890625 mean=9.1552734375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042724609375 mean=0.0013580322265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0576171875 mean=0.00010728836059570312 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037109375 mean=0.001495361328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.055908203125 mean=6.008148193359375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.036865234375 mean=0.00115966796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05224609375 mean=0.00010061264038085938 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0257568359375 mean=0.0003452301025390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=9.1552734375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.024169921875 mean=0.0002498626708984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0439453125 mean=0.0003147125244140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0208740234375 mean=-0.000354766845703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=0.00016021728515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02001953125 mean=-0.00011587142944335938 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03564453125 mean=0.0004291534423828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0216064453125 mean=-0.0002689361572265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03564453125 mean=8.726119995117188e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.023681640625 mean=-0.000911712646484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0361328125 mean=0.0002918243408203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0208740234375 mean=-0.00064849853515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036376953125 mean=0.00018215179443359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0238037109375 mean=-0.00087738037109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03369140625 mean=-9.679794311523438e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.019775390625 mean=-0.0004558563232421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.033935546875 mean=-0.0001220703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0213623046875 mean=-0.00113677978515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0380859375 mean=-0.00010728836059570312 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0240478515625 mean=-0.000751495361328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03955078125 mean=3.003515303134918e-08 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0240478515625 mean=-0.00113677978515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.040283203125 mean=0.00019741058349609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03076171875 mean=-0.00131988525390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037353515625 mean=0.00011157989501953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05322265625 mean=-0.002410888671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=8.535385131835938e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.041259765625 mean=-0.0019378662109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04296875 mean=-0.0001316070556640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0439453125 mean=0.002655029296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.95s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1200-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 81.12s +Conversion finished in 480.7664725780487 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1200-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1200-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_daemon_1200/mcore2hf-config.Ah9Wjg ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1280.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1280.log new file mode 100644 index 0000000000000000000000000000000000000000..c4e63d875658b421c33b2d2f0a91a563fa758caa --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1280.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_daemon_1280 ++ mkdir -p /tmp/mcore2hf_daemon_1280 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 1280 ']' ++ MANUAL_ITER=1280 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280 ++ CHECKPOINT_BASENAME=load_view1280 ++ [[ load_view1280 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_daemon_1280/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_daemon_1280/mcore2hf-config.wqvCEv ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_daemon_1280/mcore2hf-config.wqvCEv:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_daemon_1280/mcore2hf-config.wqvCEv:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280 +W1014 22:26:36.275000 136385 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1014 22:26:36.275000 136385 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=524288 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=524288 +++ MAX_POSITION_EMBEDDINGS=524288 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280 +W1014 22:26:51.271000 136778 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1014 22:26:51.271000 136778 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 524288 ']' ++ '[' 524288 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_daemon_1280/mcore2hf-config.wqvCEv +W1014 22:27:04.171000 137075 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1014 22:27:04.171000 137075 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_daemon_1280/mcore2hf-config.wqvCEv/config.json' +✅config.json: /tmp/mcore2hf_daemon_1280/mcore2hf-config.wqvCEv/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_daemon_1280/mcore2hf-config.wqvCEv/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 524288, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_daemon_1280/mcore2hf-config.wqvCEv ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280 1280 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280 ++ local ITER_NUM=1280 ++ '[' -z 1280 ']' +++ printf %07d 1280 ++ local ITER_BASENAME=iter_0001280 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280/iter_0001280 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280/iter_0001280 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280/latest_checkpointed_iteration.txt ++ LATEST_VALUE=1280 ++ '[' 1280 = 1280 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=1280 ++ echo -e '\033[32mLatest checkpoint iteration: 1280\033[0m\n' +Latest checkpoint iteration: 1280 + ++ I=1280 ++ [[ 1280 =~ ^[0-9]+$ ]] +++ printf %07d 1280 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280/iter_0001280 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280/iter_0001280/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280/iter_1280-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1280-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1280-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1280-hf true false bf16 /tmp/mcore2hf_daemon_1280/mcore2hf-config.wqvCEv 1280 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1280-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 50979 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_daemon_1280/mcore2hf-config.wqvCEv --hf-dir /tmp/mcore2hf_daemon_1280/mcore2hf-config.wqvCEv --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 524288 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1280-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 1280 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1014 22:27:23.932000 137717 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1014 22:27:23.932000 137717 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 1280 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_daemon_1280/mcore2hf-config.wqvCEv + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 524288 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1280-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_daemon_1280/mcore2hf-config.wqvCEv + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.136 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_wqvCEv/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_wqvCEv/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_wqvCEv.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280 at iteration 1280 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1280 [ t 1/1, p 1/1 ] at iteration 1280 +[convert] synchronizer created in 447.08s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07666015625 mean=-0.000492095947265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10205078125 mean=0.0015716552734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07861328125 mean=-0.00013446807861328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10546875 mean=0.00015354156494140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0849609375 mean=-0.0002193450927734375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07958984375 mean=0.000850677490234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.087890625 mean=-0.0002613067626953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0771484375 mean=0.00145721435546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09619140625 mean=-0.000213623046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07373046875 mean=-0.000782012939453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09130859375 mean=-8.7738037109375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.062255859375 mean=-3.62396240234375e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09375 mean=-9.298324584960938e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0625 mean=0.00010156631469726562 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.087890625 mean=3.600120544433594e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05615234375 mean=-0.0004482269287109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.083984375 mean=5.698204040527344e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.054443359375 mean=-0.0004596710205078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08251953125 mean=-3.3229589462280273e-06 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06591796875 mean=-0.0003261566162109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.00019073486328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05419921875 mean=-0.00042724609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07177734375 mean=8.726119995117188e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04296875 mean=-0.0010528564453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.00014209747314453125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04638671875 mean=-0.00099945068359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07373046875 mean=0.0002155303955078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.048828125 mean=-0.000446319580078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.072265625 mean=0.0002841949462890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.047119140625 mean=-0.0005035400390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06982421875 mean=0.000316619873046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.046142578125 mean=-0.00151824951171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06591796875 mean=0.0002574920654296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.046630859375 mean=-0.00180816650390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06884765625 mean=0.00054168701171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04931640625 mean=-0.0017242431640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06787109375 mean=0.0003376007080078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05029296875 mean=-0.00121307373046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060546875 mean=0.000354766845703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.041015625 mean=-0.0021209716796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0556640625 mean=0.00028228759765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04345703125 mean=-0.0030059814453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.053466796875 mean=0.00041961669921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037109375 mean=-0.00213623046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05224609375 mean=0.000423431396484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.034912109375 mean=-0.0024261474609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049560546875 mean=0.0003757476806640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.033447265625 mean=-0.00164031982421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=0.000301361083984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.031005859375 mean=-0.001495361328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.051513671875 mean=0.0004062652587890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.030029296875 mean=-0.00177764892578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.047607421875 mean=0.000370025634765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0267333984375 mean=-0.00080108642578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04833984375 mean=0.000518798828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.024169921875 mean=-0.000873565673828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.051513671875 mean=0.0008087158203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.025146484375 mean=-0.00083160400390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048828125 mean=0.000537872314453125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.026123046875 mean=-0.000614166259765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.046142578125 mean=0.0001373291015625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0252685546875 mean=-0.0003719329833984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=8.058547973632812e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.027587890625 mean=-1.4185905456542969e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.0001430511474609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02685546875 mean=4.220008850097656e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05712890625 mean=4.029273986816406e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0322265625 mean=0.0003337860107421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05908203125 mean=3.910064697265625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04296875 mean=0.0010528564453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.058837890625 mean=8.96453857421875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042724609375 mean=0.00135040283203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0576171875 mean=0.000102996826171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037109375 mean=0.00154876708984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.055908203125 mean=6.246566772460938e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.036865234375 mean=0.00122833251953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05224609375 mean=9.584426879882812e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0257568359375 mean=0.00034332275390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=9.34600830078125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0240478515625 mean=0.0002155303955078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0439453125 mean=0.0003108978271484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.020751953125 mean=-0.0003643035888671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0380859375 mean=0.0001583099365234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02001953125 mean=-0.00023174285888671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.035400390625 mean=0.0004367828369140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.021728515625 mean=-0.000331878662109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03564453125 mean=9.34600830078125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.023681640625 mean=-0.0008544921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0361328125 mean=0.000293731689453125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.020751953125 mean=-0.0005950927734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036376953125 mean=0.00018405914306640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0240478515625 mean=-0.000858306884765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03369140625 mean=-8.7738037109375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0198974609375 mean=-0.0004673004150390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.033935546875 mean=-0.00013065338134765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.021484375 mean=-0.00115203857421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0380859375 mean=-0.000110626220703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0238037109375 mean=-0.000827789306640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03955078125 mean=1.6540288925170898e-06 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0240478515625 mean=-0.0010833740234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0400390625 mean=0.00019931793212890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0308837890625 mean=-0.00121307373046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037353515625 mean=0.0001049041748046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05322265625 mean=-0.00244140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=8.296966552734375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.041259765625 mean=-0.00189971923828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04296875 mean=-0.0001354217529296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0439453125 mean=0.00262451171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.84s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1280-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 73.50s +Conversion finished in 522.7259006500244 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1280-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1280-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_daemon_1280/mcore2hf-config.wqvCEv ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1360.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1360.log new file mode 100644 index 0000000000000000000000000000000000000000..ffcf87c1ce69d8dc15998622f611266e69420a68 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1360.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_daemon_1360 ++ mkdir -p /tmp/mcore2hf_daemon_1360 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 1360 ']' ++ MANUAL_ITER=1360 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360 ++ CHECKPOINT_BASENAME=load_view1360 ++ [[ load_view1360 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_daemon_1360/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_daemon_1360/mcore2hf-config.ix83bw ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_daemon_1360/mcore2hf-config.ix83bw:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_daemon_1360/mcore2hf-config.ix83bw:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360 +W1015 03:28:42.623000 463985 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1015 03:28:42.623000 463985 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=524288 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=524288 +++ MAX_POSITION_EMBEDDINGS=524288 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360 +W1015 03:28:57.616000 464444 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1015 03:28:57.616000 464444 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 524288 ']' ++ '[' 524288 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_daemon_1360/mcore2hf-config.ix83bw +W1015 03:29:10.112000 464659 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1015 03:29:10.112000 464659 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_daemon_1360/mcore2hf-config.ix83bw/config.json' +✅config.json: /tmp/mcore2hf_daemon_1360/mcore2hf-config.ix83bw/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_daemon_1360/mcore2hf-config.ix83bw/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 524288, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_daemon_1360/mcore2hf-config.ix83bw ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360 1360 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360 ++ local ITER_NUM=1360 ++ '[' -z 1360 ']' +++ printf %07d 1360 ++ local ITER_BASENAME=iter_0001360 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360/iter_0001360 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360/iter_0001360 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360/latest_checkpointed_iteration.txt ++ LATEST_VALUE=1360 ++ '[' 1360 = 1360 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=1360 ++ echo -e '\033[32mLatest checkpoint iteration: 1360\033[0m\n' +Latest checkpoint iteration: 1360 + ++ I=1360 ++ [[ 1360 =~ ^[0-9]+$ ]] +++ printf %07d 1360 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360/iter_0001360 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360/iter_0001360/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360/iter_1360-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1360-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1360-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1360-hf true false bf16 /tmp/mcore2hf_daemon_1360/mcore2hf-config.ix83bw 1360 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1360-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 42497 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_daemon_1360/mcore2hf-config.ix83bw --hf-dir /tmp/mcore2hf_daemon_1360/mcore2hf-config.ix83bw --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 524288 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1360-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 1360 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1015 03:29:30.009000 465107 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1015 03:29:30.009000 465107 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 1360 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_daemon_1360/mcore2hf-config.ix83bw + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 524288 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1360-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_daemon_1360/mcore2hf-config.ix83bw + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.129 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_ix83bw/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_ix83bw/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_ix83bw.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360 at iteration 1360 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1360 [ t 1/1, p 1/1 ] at iteration 1360 +[convert] synchronizer created in 375.82s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07666015625 mean=-0.00048828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10205078125 mean=0.001556396484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07861328125 mean=-0.0001316070556640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10546875 mean=0.00010824203491210938 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08447265625 mean=-0.00021076202392578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07958984375 mean=0.00079345703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.087890625 mean=-0.0002536773681640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0771484375 mean=0.0014190673828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09619140625 mean=-0.000213623046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07373046875 mean=-0.000705718994140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09130859375 mean=-8.153915405273438e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.062255859375 mean=-6.246566772460938e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09326171875 mean=-9.298324584960938e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0625 mean=6.67572021484375e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.087890625 mean=3.5762786865234375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05615234375 mean=-0.00028228759765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.083984375 mean=5.6743621826171875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.054443359375 mean=-0.00032806396484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08203125 mean=-5.27501106262207e-06 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0654296875 mean=-0.000354766845703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.0001964569091796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05419921875 mean=-0.0004425048828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07177734375 mean=8.630752563476562e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042724609375 mean=-0.0010528564453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.0001392364501953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04638671875 mean=-0.0009765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07373046875 mean=0.000217437744140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.048828125 mean=-0.0003986358642578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.072265625 mean=0.0002841949462890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.046875 mean=-0.0004367828369140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06982421875 mean=0.000316619873046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0458984375 mean=-0.00160980224609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06591796875 mean=0.000255584716796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04638671875 mean=-0.00186920166015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06884765625 mean=0.000545501708984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.049072265625 mean=-0.001739501953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06787109375 mean=0.000335693359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.050048828125 mean=-0.00128173828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060546875 mean=0.0003528594970703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.040771484375 mean=-0.002105712890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0556640625 mean=0.00028228759765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043212890625 mean=-0.00299072265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.053466796875 mean=0.0004177093505859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.036865234375 mean=-0.0021514892578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05224609375 mean=0.0004215240478515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03466796875 mean=-0.0023651123046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049560546875 mean=0.0003719329833984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.033203125 mean=-0.00167083740234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=0.0002994537353515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0308837890625 mean=-0.00145721435546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.051513671875 mean=0.0003948211669921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.030029296875 mean=-0.00170135498046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.047607421875 mean=0.0003719329833984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.026611328125 mean=-0.00080108642578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04833984375 mean=0.00051116943359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02392578125 mean=-0.0008087158203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.00080108642578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0250244140625 mean=-0.000743865966796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048828125 mean=0.0005340576171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0260009765625 mean=-0.00052642822265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.046142578125 mean=0.0001316070556640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.025146484375 mean=-0.00032806396484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=8.0108642578125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.027587890625 mean=3.9577484130859375e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.00014019012451171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.026611328125 mean=0.00012969970703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05712890625 mean=3.981590270996094e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0322265625 mean=0.0003948211669921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05908203125 mean=3.719329833984375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04296875 mean=0.00110626220703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05859375 mean=8.7738037109375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042724609375 mean=0.00138092041015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.057373046875 mean=9.965896606445312e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037109375 mean=0.001617431640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0556640625 mean=6.151199340820312e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.036865234375 mean=0.00131988525390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05224609375 mean=9.298324584960938e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.025634765625 mean=0.000484466552734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048828125 mean=9.107589721679688e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0240478515625 mean=0.0002651214599609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0439453125 mean=0.000308990478515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.020751953125 mean=-0.00034332275390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0380859375 mean=0.000156402587890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02001953125 mean=-0.0002460479736328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.035400390625 mean=0.00043487548828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.021484375 mean=-0.0003032684326171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03564453125 mean=9.5367431640625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0234375 mean=-0.000835418701171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0361328125 mean=0.0002918243408203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.020751953125 mean=-0.000583648681640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036376953125 mean=0.00018024444580078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.023681640625 mean=-0.000827789306640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03369140625 mean=-8.440017700195312e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0198974609375 mean=-0.00055694580078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0341796875 mean=-0.0001373291015625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0213623046875 mean=-0.00116729736328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0380859375 mean=-0.00011730194091796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02392578125 mean=-0.00067901611328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.039794921875 mean=-6.9141387939453125e-06 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0240478515625 mean=-0.0010833740234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0400390625 mean=0.00019168853759765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03076171875 mean=-0.0012664794921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037109375 mean=0.00010442733764648438 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05322265625 mean=-0.0025177001953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=7.581710815429688e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.041015625 mean=-0.00179290771484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04296875 mean=-0.00013637542724609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043701171875 mean=0.0025634765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.86s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1360-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 74.30s +Conversion finished in 452.27539348602295 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1360-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1360-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_daemon_1360/mcore2hf-config.ix83bw ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1520.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1520.log new file mode 100644 index 0000000000000000000000000000000000000000..c74492c383ffacc7768c01d4ea391c845b9f729d --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1520.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_daemon_1520 ++ mkdir -p /tmp/mcore2hf_daemon_1520 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 1520 ']' ++ MANUAL_ITER=1520 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520 ++ CHECKPOINT_BASENAME=load_view1520 ++ [[ load_view1520 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_daemon_1520/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_daemon_1520/mcore2hf-config.0uxk17 ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_daemon_1520/mcore2hf-config.0uxk17:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_daemon_1520/mcore2hf-config.0uxk17:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520 +W1015 13:30:27.726000 113046 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1015 13:30:27.726000 113046 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=524288 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=524288 +++ MAX_POSITION_EMBEDDINGS=524288 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520 +W1015 13:30:42.762000 113455 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1015 13:30:42.762000 113455 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 524288 ']' ++ '[' 524288 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_daemon_1520/mcore2hf-config.0uxk17 +W1015 13:30:56.085000 113902 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1015 13:30:56.085000 113902 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_daemon_1520/mcore2hf-config.0uxk17/config.json' +✅config.json: /tmp/mcore2hf_daemon_1520/mcore2hf-config.0uxk17/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_daemon_1520/mcore2hf-config.0uxk17/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 524288, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_daemon_1520/mcore2hf-config.0uxk17 ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520 1520 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520 ++ local ITER_NUM=1520 ++ '[' -z 1520 ']' +++ printf %07d 1520 ++ local ITER_BASENAME=iter_0001520 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520/iter_0001520 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520/iter_0001520 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520/latest_checkpointed_iteration.txt ++ LATEST_VALUE=1520 ++ '[' 1520 = 1520 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=1520 ++ echo -e '\033[32mLatest checkpoint iteration: 1520\033[0m\n' +Latest checkpoint iteration: 1520 + ++ I=1520 ++ [[ 1520 =~ ^[0-9]+$ ]] +++ printf %07d 1520 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520/iter_0001520 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520/iter_0001520/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520/iter_1520-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1520-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1520-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1520-hf true false bf16 /tmp/mcore2hf_daemon_1520/mcore2hf-config.0uxk17 1520 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1520-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 60747 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_daemon_1520/mcore2hf-config.0uxk17 --hf-dir /tmp/mcore2hf_daemon_1520/mcore2hf-config.0uxk17 --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 524288 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1520-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 1520 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1015 13:31:16.006000 114299 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1015 13:31:16.006000 114299 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 1520 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_daemon_1520/mcore2hf-config.0uxk17 + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 524288 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1520-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_daemon_1520/mcore2hf-config.0uxk17 + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.132 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_0uxk17/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_0uxk17/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_0uxk17.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520 at iteration 1520 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1520 [ t 1/1, p 1/1 ] at iteration 1520 +[convert] synchronizer created in 370.29s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07666015625 mean=-0.0004749298095703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.1015625 mean=0.00144195556640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07861328125 mean=-0.0001316070556640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.1044921875 mean=5.364418029785156e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08447265625 mean=-0.00021457672119140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0791015625 mean=0.0009613037109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.087890625 mean=-0.0002593994140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.076171875 mean=0.001495361328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.095703125 mean=-0.00022125244140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0732421875 mean=-0.000797271728515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0908203125 mean=-8.392333984375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06201171875 mean=-0.00024127960205078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09326171875 mean=-9.72747802734375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.061767578125 mean=-1.3589859008789062e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.087890625 mean=3.647804260253906e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0556640625 mean=-0.000492095947265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.083984375 mean=5.745887756347656e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.053955078125 mean=-0.0003509521484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08203125 mean=-3.4868717193603516e-06 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0654296875 mean=-0.000652313232421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.0001983642578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0537109375 mean=-0.00041961669921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07177734375 mean=7.963180541992188e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042724609375 mean=-0.001068115234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.0001316070556640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04638671875 mean=-0.0008087158203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07373046875 mean=0.0002155303955078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04833984375 mean=-0.0003509521484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.072265625 mean=0.0002765655517578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04638671875 mean=-0.0004749298095703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06982421875 mean=0.0003147125244140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.045654296875 mean=-0.00154876708984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06591796875 mean=0.0002651214599609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04541015625 mean=-0.00189971923828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06884765625 mean=0.000545501708984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.048583984375 mean=-0.00164794921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06787109375 mean=0.000339508056640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0498046875 mean=-0.0012054443359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060546875 mean=0.0003452301025390625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0400390625 mean=-0.0022430419921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0556640625 mean=0.0002803802490234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04296875 mean=-0.0029144287109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.053466796875 mean=0.0004177093505859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03662109375 mean=-0.0021209716796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05224609375 mean=0.000415802001953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.034423828125 mean=-0.0023956298828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049560546875 mean=0.000377655029296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03271484375 mean=-0.00154876708984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=0.00029754638671875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0303955078125 mean=-0.00151824951171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.051513671875 mean=0.0003986358642578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0299072265625 mean=-0.0016632080078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.047607421875 mean=0.00037384033203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0263671875 mean=-0.000728607177734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04833984375 mean=0.0005035400390625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.023681640625 mean=-0.00074005126953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.0008087158203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0247802734375 mean=-0.000759124755859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048828125 mean=0.00054168701171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0260009765625 mean=-0.000644683837890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.046142578125 mean=0.0001373291015625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02490234375 mean=-0.0001983642578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049560546875 mean=7.915496826171875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0274658203125 mean=2.014636993408203e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.051513671875 mean=0.0001392364501953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0263671875 mean=0.00014495849609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05712890625 mean=3.790855407714844e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.031982421875 mean=0.000270843505859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05908203125 mean=3.218650817871094e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042724609375 mean=0.0010223388671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05859375 mean=8.106231689453125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04248046875 mean=0.001373291015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.057373046875 mean=9.775161743164062e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.036865234375 mean=0.0016021728515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0556640625 mean=5.936622619628906e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.036865234375 mean=0.00131988525390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05224609375 mean=8.96453857421875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0255126953125 mean=0.0004367828369140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048828125 mean=8.535385131835938e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.023681640625 mean=0.00026702880859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0439453125 mean=0.00030517578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0206298828125 mean=-0.000457763671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0380859375 mean=0.00016021728515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0196533203125 mean=-0.00014781951904296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.035400390625 mean=0.000446319580078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0211181640625 mean=-0.000362396240234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03564453125 mean=9.202957153320312e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0233154296875 mean=-0.00074005126953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.035888671875 mean=0.000286102294921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0205078125 mean=-0.0004825592041015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036376953125 mean=0.0001850128173828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0235595703125 mean=-0.000904083251953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03369140625 mean=-7.772445678710938e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0196533203125 mean=-0.0004730224609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0341796875 mean=-0.00014591217041015625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02099609375 mean=-0.0009613037109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0380859375 mean=-0.00011920928955078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0238037109375 mean=-0.000751495361328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.039794921875 mean=-9.953975677490234e-06 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02392578125 mean=-0.00104522705078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0400390625 mean=0.00019359588623046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.030517578125 mean=-0.00127410888671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037109375 mean=9.34600830078125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05322265625 mean=-0.0024871826171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=7.390975952148438e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.041015625 mean=-0.001953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.042724609375 mean=-0.0001430511474609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043701171875 mean=0.002655029296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.85s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1520-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 72.13s +Conversion finished in 444.6920564174652 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1520-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1520-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_daemon_1520/mcore2hf-config.0uxk17 ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter160.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter160.log new file mode 100644 index 0000000000000000000000000000000000000000..bd7f9288b81c8459bae45b3c0600ddd69b730cc6 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter160.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_daemon_160 ++ mkdir -p /tmp/mcore2hf_daemon_160 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 160 ']' ++ MANUAL_ITER=160 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160 ++ CHECKPOINT_BASENAME=load_view160 ++ [[ load_view160 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_daemon_160/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_daemon_160/mcore2hf-config.4coICl ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_daemon_160/mcore2hf-config.4coICl:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_daemon_160/mcore2hf-config.4coICl:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160 +W1011 23:55:39.735000 258035 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 23:55:39.735000 258035 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=524288 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=524288 +++ MAX_POSITION_EMBEDDINGS=524288 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160 +W1011 23:55:54.742000 258532 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 23:55:54.742000 258532 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 524288 ']' ++ '[' 524288 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_daemon_160/mcore2hf-config.4coICl +W1011 23:56:07.240000 258732 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 23:56:07.240000 258732 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_daemon_160/mcore2hf-config.4coICl/config.json' +✅config.json: /tmp/mcore2hf_daemon_160/mcore2hf-config.4coICl/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_daemon_160/mcore2hf-config.4coICl/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 524288, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_daemon_160/mcore2hf-config.4coICl ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160 160 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160 ++ local ITER_NUM=160 ++ '[' -z 160 ']' +++ printf %07d 160 ++ local ITER_BASENAME=iter_0000160 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160/iter_0000160 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160/iter_0000160 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160/latest_checkpointed_iteration.txt ++ LATEST_VALUE=160 ++ '[' 160 = 160 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=160 ++ echo -e '\033[32mLatest checkpoint iteration: 160\033[0m\n' +Latest checkpoint iteration: 160 + ++ I=160 ++ [[ 160 =~ ^[0-9]+$ ]] +++ printf %07d 160 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160/iter_0000160 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160/iter_0000160/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160/iter_160-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_160-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_160-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_160-hf true false bf16 /tmp/mcore2hf_daemon_160/mcore2hf-config.4coICl 160 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_160-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 39881 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_daemon_160/mcore2hf-config.4coICl --hf-dir /tmp/mcore2hf_daemon_160/mcore2hf-config.4coICl --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 524288 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_160-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 160 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1011 23:56:27.001000 259206 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 23:56:27.001000 259206 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 160 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_daemon_160/mcore2hf-config.4coICl + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 524288 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_160-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_daemon_160/mcore2hf-config.4coICl + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.138 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_4coICl/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_4coICl/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_4coICl.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160 at iteration 160 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view160 [ t 1/1, p 1/1 ] at iteration 160 +[convert] synchronizer created in 387.80s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07763671875 mean=-0.000507354736328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.1044921875 mean=0.00176239013671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.080078125 mean=-0.000164031982421875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.1083984375 mean=-7.867813110351562e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08642578125 mean=-0.0002689361572265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.080078125 mean=0.000858306884765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09033203125 mean=-0.0002117156982421875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.080078125 mean=0.0014190673828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09814453125 mean=-0.00023555755615234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07470703125 mean=-0.000827789306640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09326171875 mean=-7.43865966796875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06298828125 mean=-0.00023651123046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.095703125 mean=-7.486343383789062e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0634765625 mean=-5.990266799926758e-06 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08984375 mean=3.147125244140625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.056396484375 mean=-0.0002536773681640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0859375 mean=2.384185791015625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05419921875 mean=-0.0003643035888671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.083984375 mean=7.2479248046875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06591796875 mean=-0.00048828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07763671875 mean=0.0002231597900390625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.053955078125 mean=-0.000553131103515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0732421875 mean=0.000148773193359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04345703125 mean=-0.0009918212890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07763671875 mean=0.0001621246337890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.046630859375 mean=-0.000804901123046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0751953125 mean=0.0002307891845703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.049560546875 mean=-0.0010223388671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07373046875 mean=0.00029754638671875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04833984375 mean=-0.00087738037109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07080078125 mean=0.00030517578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04736328125 mean=-0.001373291015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06689453125 mean=0.000278472900390625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0478515625 mean=-0.0023040771484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06982421875 mean=0.000606536865234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0498046875 mean=-0.001617431640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.068359375 mean=0.00037384033203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05029296875 mean=-0.00107574462890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.061279296875 mean=0.000408172607421875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.041748046875 mean=-0.002288818359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.056396484375 mean=0.000339508056640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.044189453125 mean=-0.0030517578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0546875 mean=0.000461578369140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037841796875 mean=-0.0021209716796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.053466796875 mean=0.00043487548828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.035400390625 mean=-0.0025634765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.050537109375 mean=0.000400543212890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.033935546875 mean=-0.001800537109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.050048828125 mean=0.000324249267578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03173828125 mean=-0.0018463134765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052490234375 mean=0.00045013427734375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0302734375 mean=-0.001708984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048828125 mean=0.000408172607421875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0274658203125 mean=-0.0009765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04931640625 mean=0.00054931640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0238037109375 mean=-0.0009613037109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052490234375 mean=0.0008544921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.025390625 mean=-0.000957489013671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.050048828125 mean=0.0005340576171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.026123046875 mean=-0.000606536865234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04736328125 mean=0.00013637542724609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.025146484375 mean=-0.0001621246337890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05078125 mean=9.393692016601562e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.027587890625 mean=2.300739288330078e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052978515625 mean=0.0001811981201171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0269775390625 mean=3.2901763916015625e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05859375 mean=6.866455078125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.031982421875 mean=0.000354766845703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060546875 mean=5.5789947509765625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04296875 mean=0.00102996826171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060302734375 mean=0.00011157989501953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042724609375 mean=0.0012969970703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.058837890625 mean=0.0001220703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037109375 mean=0.0015869140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.057373046875 mean=7.963180541992188e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03662109375 mean=0.00124359130859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0537109375 mean=0.00013256072998046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02587890625 mean=0.0002231597900390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05029296875 mean=0.0001316070556640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0242919921875 mean=0.00012683868408203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.045166015625 mean=0.0003662109375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.020751953125 mean=-0.00054168701171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.039306640625 mean=0.00019550323486328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.020263671875 mean=-0.0003414154052734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03662109375 mean=0.000423431396484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.021484375 mean=-0.00031280517578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037109375 mean=0.00012159347534179688 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0234375 mean=-0.000835418701171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037353515625 mean=0.0003452301025390625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.020751953125 mean=-0.000598907470703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037353515625 mean=0.00020885467529296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0233154296875 mean=-0.00074005126953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.034912109375 mean=-6.866455078125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0196533203125 mean=-0.000583648681640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03466796875 mean=-0.000102996826171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.021484375 mean=-0.00107574462890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0380859375 mean=-9.965896606445312e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02294921875 mean=-0.00086212158203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=3.552436828613281e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0230712890625 mean=-0.00112152099609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04052734375 mean=0.0002231597900390625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0296630859375 mean=-0.001251220703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03857421875 mean=9.1552734375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05078125 mean=-0.0023040771484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03955078125 mean=6.341934204101562e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03857421875 mean=-0.0019378662109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.044189453125 mean=-0.00014495849609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042236328125 mean=0.0027008056640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.93s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_160-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 81.22s +Conversion finished in 471.30603432655334 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_160-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_160-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_daemon_160/mcore2hf-config.4coICl ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1600.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1600.log new file mode 100644 index 0000000000000000000000000000000000000000..b2c8251e165b8b43f69dfc00c5cbb25d5db6e2e4 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1600.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_daemon_1600 ++ mkdir -p /tmp/mcore2hf_daemon_1600 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 1600 ']' ++ MANUAL_ITER=1600 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600 ++ CHECKPOINT_BASENAME=load_view1600 ++ [[ load_view1600 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_daemon_1600/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_daemon_1600/mcore2hf-config.cAFVQD ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_daemon_1600/mcore2hf-config.cAFVQD:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_daemon_1600/mcore2hf-config.cAFVQD:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600 +W1015 18:32:35.875000 529757 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1015 18:32:35.875000 529757 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=524288 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=524288 +++ MAX_POSITION_EMBEDDINGS=524288 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600 +W1015 18:32:51.149000 530430 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1015 18:32:51.149000 530430 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 524288 ']' ++ '[' 524288 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_daemon_1600/mcore2hf-config.cAFVQD +W1015 18:33:04.488000 530728 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1015 18:33:04.488000 530728 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_daemon_1600/mcore2hf-config.cAFVQD/config.json' +✅config.json: /tmp/mcore2hf_daemon_1600/mcore2hf-config.cAFVQD/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_daemon_1600/mcore2hf-config.cAFVQD/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 524288, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_daemon_1600/mcore2hf-config.cAFVQD ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600 1600 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600 ++ local ITER_NUM=1600 ++ '[' -z 1600 ']' +++ printf %07d 1600 ++ local ITER_BASENAME=iter_0001600 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600/iter_0001600 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600/iter_0001600 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600/latest_checkpointed_iteration.txt ++ LATEST_VALUE=1600 ++ '[' 1600 = 1600 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=1600 ++ echo -e '\033[32mLatest checkpoint iteration: 1600\033[0m\n' +Latest checkpoint iteration: 1600 + ++ I=1600 ++ [[ 1600 =~ ^[0-9]+$ ]] +++ printf %07d 1600 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600/iter_0001600 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600/iter_0001600/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600/iter_1600-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1600-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1600-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1600-hf true false bf16 /tmp/mcore2hf_daemon_1600/mcore2hf-config.cAFVQD 1600 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1600-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 52769 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_daemon_1600/mcore2hf-config.cAFVQD --hf-dir /tmp/mcore2hf_daemon_1600/mcore2hf-config.cAFVQD --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 524288 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1600-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 1600 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1015 18:33:24.973000 531362 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1015 18:33:24.973000 531362 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 1600 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_daemon_1600/mcore2hf-config.cAFVQD + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 524288 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1600-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_daemon_1600/mcore2hf-config.cAFVQD + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.137 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_cAFVQD/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_cAFVQD/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_cAFVQD.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600 at iteration 1600 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1600 [ t 1/1, p 1/1 ] at iteration 1600 +[convert] synchronizer created in 381.87s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07666015625 mean=-0.000484466552734375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10107421875 mean=0.00146484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07861328125 mean=-0.00013637542724609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.1044921875 mean=9.250640869140625e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08447265625 mean=-0.000213623046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07861328125 mean=0.00103759765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.087890625 mean=-0.000263214111328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07568359375 mean=0.0015106201171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.095703125 mean=-0.00022220611572265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07275390625 mean=-0.0007476806640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0908203125 mean=-8.916854858398438e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.061767578125 mean=-0.000263214111328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09326171875 mean=-9.870529174804688e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0615234375 mean=8.726119995117188e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.087890625 mean=3.910064697265625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.055419921875 mean=-0.000423431396484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.083984375 mean=5.507469177246094e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0537109375 mean=-0.0002765655517578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08203125 mean=-7.599592208862305e-06 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06494140625 mean=-0.000751495361328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.0001983642578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.053466796875 mean=-0.00038909912109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07177734375 mean=7.724761962890625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042724609375 mean=-0.00102996826171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.00012969970703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0458984375 mean=-0.00084686279296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07373046875 mean=0.00021648406982421875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.048095703125 mean=-0.0003032684326171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.072265625 mean=0.0002803802490234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.046142578125 mean=-0.00045013427734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06982421875 mean=0.000316619873046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.044921875 mean=-0.0013885498046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06591796875 mean=0.0002651214599609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.044921875 mean=-0.00188446044921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06884765625 mean=0.00054931640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04833984375 mean=-0.00162506103515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06787109375 mean=0.000347137451171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04931640625 mean=-0.001129150390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060546875 mean=0.0003452301025390625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.039794921875 mean=-0.002166748046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.055908203125 mean=0.00028228759765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04248046875 mean=-0.002899169921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.053466796875 mean=0.000415802001953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0361328125 mean=-0.0020599365234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05224609375 mean=0.00041961669921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0341796875 mean=-0.002288818359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049560546875 mean=0.0003795623779296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.032470703125 mean=-0.0015716552734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=0.00029754638671875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0301513671875 mean=-0.00151824951171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.051513671875 mean=0.000392913818359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0296630859375 mean=-0.00157928466796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.047607421875 mean=0.0003681182861328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0263671875 mean=-0.00070953369140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04833984375 mean=0.00051116943359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0235595703125 mean=-0.000701904296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.0008087158203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.024658203125 mean=-0.000743865966796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048828125 mean=0.000545501708984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02587890625 mean=-0.00052642822265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04638671875 mean=0.0001373291015625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02490234375 mean=-0.00016021728515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049560546875 mean=8.0108642578125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0272216796875 mean=-3.1888484954833984e-06 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.051513671875 mean=0.000141143798828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0263671875 mean=0.0001659393310546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.056884765625 mean=4.172325134277344e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03173828125 mean=0.0003566741943359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.058837890625 mean=3.337860107421875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04248046875 mean=0.0009613037109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05859375 mean=8.249282836914062e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04248046875 mean=0.00148773193359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.057373046875 mean=0.00010061264038085938 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.036865234375 mean=0.00154876708984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0556640625 mean=5.8650970458984375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03662109375 mean=0.00141143798828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05224609375 mean=8.821487426757812e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.025390625 mean=0.000392913818359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048828125 mean=8.678436279296875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0235595703125 mean=0.0002651214599609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0439453125 mean=0.0003032684326171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0205078125 mean=-0.00048065185546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0380859375 mean=0.00016021728515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.01953125 mean=-0.00018596649169921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03564453125 mean=0.0004425048828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0211181640625 mean=-0.0004520416259765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03564453125 mean=9.489059448242188e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0230712890625 mean=-0.00070953369140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.035888671875 mean=0.000274658203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0203857421875 mean=-0.000553131103515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0361328125 mean=0.00018978118896484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0233154296875 mean=-0.0007781982421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03369140625 mean=-7.867813110351562e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0196533203125 mean=-0.000499725341796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0341796875 mean=-0.00014209747314453125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0208740234375 mean=-0.000957489013671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0380859375 mean=-0.00012493133544921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0235595703125 mean=-0.00067138671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.039794921875 mean=-1.4901161193847656e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0238037109375 mean=-0.00103759765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0400390625 mean=0.0001888275146484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0302734375 mean=-0.001434326171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037109375 mean=9.441375732421875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.052978515625 mean=-0.0024871826171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=7.2479248046875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.040771484375 mean=-0.00201416015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04296875 mean=-0.0001392364501953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043701171875 mean=0.0025482177734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.87s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1600-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 71.57s +Conversion finished in 455.67713832855225 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1600-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1600-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_daemon_1600/mcore2hf-config.cAFVQD ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1760.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1760.log new file mode 100644 index 0000000000000000000000000000000000000000..6279252699559da12488dfb93d25039dd231a1e3 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1760.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_daemon_1760 ++ mkdir -p /tmp/mcore2hf_daemon_1760 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 1760 ']' ++ MANUAL_ITER=1760 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760 ++ CHECKPOINT_BASENAME=load_view1760 ++ [[ load_view1760 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_daemon_1760/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_daemon_1760/mcore2hf-config.uSVJKk ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_daemon_1760/mcore2hf-config.uSVJKk:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_daemon_1760/mcore2hf-config.uSVJKk:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760 +W1016 04:37:01.726000 207966 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1016 04:37:01.726000 207966 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=524288 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=524288 +++ MAX_POSITION_EMBEDDINGS=524288 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760 +W1016 04:37:14.222000 208267 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1016 04:37:14.222000 208267 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 524288 ']' ++ '[' 524288 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_daemon_1760/mcore2hf-config.uSVJKk +W1016 04:37:26.903000 208538 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1016 04:37:26.903000 208538 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_daemon_1760/mcore2hf-config.uSVJKk/config.json' +✅config.json: /tmp/mcore2hf_daemon_1760/mcore2hf-config.uSVJKk/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_daemon_1760/mcore2hf-config.uSVJKk/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 524288, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_daemon_1760/mcore2hf-config.uSVJKk ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760 1760 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760 ++ local ITER_NUM=1760 ++ '[' -z 1760 ']' +++ printf %07d 1760 ++ local ITER_BASENAME=iter_0001760 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760/iter_0001760 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760/iter_0001760 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760/latest_checkpointed_iteration.txt ++ LATEST_VALUE=1760 ++ '[' 1760 = 1760 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=1760 ++ echo -e '\033[32mLatest checkpoint iteration: 1760\033[0m\n' +Latest checkpoint iteration: 1760 + ++ I=1760 ++ [[ 1760 =~ ^[0-9]+$ ]] +++ printf %07d 1760 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760/iter_0001760 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760/iter_0001760/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760/iter_1760-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1760-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1760-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1760-hf true false bf16 /tmp/mcore2hf_daemon_1760/mcore2hf-config.uSVJKk 1760 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1760-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 50731 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_daemon_1760/mcore2hf-config.uSVJKk --hf-dir /tmp/mcore2hf_daemon_1760/mcore2hf-config.uSVJKk --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 524288 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1760-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 1760 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1016 04:37:46.902000 209096 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1016 04:37:46.902000 209096 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 1760 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_daemon_1760/mcore2hf-config.uSVJKk + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 524288 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1760-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_daemon_1760/mcore2hf-config.uSVJKk + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.127 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_uSVJKk/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_uSVJKk/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_uSVJKk.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760 at iteration 1760 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1760 [ t 1/1, p 1/1 ] at iteration 1760 +[convert] synchronizer created in 373.55s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07666015625 mean=-0.000492095947265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10107421875 mean=0.00144195556640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07861328125 mean=-0.000125885009765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10400390625 mean=6.616115570068359e-06 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08447265625 mean=-0.00020885467529296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.078125 mean=0.00106048583984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.087890625 mean=-0.0002727508544921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0751953125 mean=0.00164031982421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.095703125 mean=-0.0002269744873046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.072265625 mean=-0.000812530517578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0908203125 mean=-9.918212890625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.061279296875 mean=-0.00028228759765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09326171875 mean=-0.00010633468627929688 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06103515625 mean=-1.1861324310302734e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.087890625 mean=3.9577484130859375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.054931640625 mean=-0.000415802001953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.083984375 mean=5.364418029785156e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.052978515625 mean=-0.0002841949462890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08203125 mean=-2.1219253540039062e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.064453125 mean=-0.000743865966796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.000194549560546875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.052734375 mean=-0.00028228759765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07177734375 mean=7.62939453125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04248046875 mean=-0.00083160400390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.0001277923583984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.045166015625 mean=-0.000812530517578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07373046875 mean=0.000217437744140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.047607421875 mean=-0.0003604888916015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.072265625 mean=0.0002880096435546875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04541015625 mean=-0.000518798828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06982421875 mean=0.0003223419189453125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04443359375 mean=-0.0014495849609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06591796875 mean=0.00026702880859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04443359375 mean=-0.0018463134765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06884765625 mean=0.00054931640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.048095703125 mean=-0.00164794921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06787109375 mean=0.0003528594970703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.048828125 mean=-0.0010833740234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060546875 mean=0.0003414154052734375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03955078125 mean=-0.002105712890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.055908203125 mean=0.000286102294921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0419921875 mean=-0.0028228759765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0537109375 mean=0.0004215240478515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03564453125 mean=-0.002105712890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052490234375 mean=0.0004177093505859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03369140625 mean=-0.0022125244140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=0.0003814697265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.031982421875 mean=-0.0014190673828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=0.000301361083984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0296630859375 mean=-0.001495361328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.051513671875 mean=0.0003910064697265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.029052734375 mean=-0.00150299072265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.047607421875 mean=0.0003662109375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0260009765625 mean=-0.00064849853515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04833984375 mean=0.000518798828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.023681640625 mean=-0.000682830810546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.0008087158203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0242919921875 mean=-0.000881195068359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=0.000545501708984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0255126953125 mean=-0.000431060791015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04638671875 mean=0.00014209747314453125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0247802734375 mean=-0.00010395050048828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=8.0108642578125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.027099609375 mean=0.00014972686767578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.00013256072998046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.026123046875 mean=0.00019359588623046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.056884765625 mean=4.1484832763671875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03173828125 mean=0.000446319580078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.058837890625 mean=3.2901763916015625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042236328125 mean=0.00096893310546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05859375 mean=7.867813110351562e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04248046875 mean=0.00135040283203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.057373046875 mean=9.679794311523438e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.036865234375 mean=0.001708984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0556640625 mean=5.793571472167969e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03662109375 mean=0.00136566162109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052001953125 mean=8.916854858398438e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0252685546875 mean=0.000492095947265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048828125 mean=8.678436279296875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0234375 mean=0.0001468658447265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0439453125 mean=0.0003032684326171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0201416015625 mean=-0.0002880096435546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0380859375 mean=0.0001621246337890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0191650390625 mean=-0.00015163421630859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03564453125 mean=0.000446319580078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02099609375 mean=-0.000446319580078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03564453125 mean=9.107589721679688e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0230712890625 mean=-0.000797271728515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.035888671875 mean=0.0002613067626953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0203857421875 mean=-0.0005950927734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0361328125 mean=0.00019168853759765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.023193359375 mean=-0.0008544921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03369140625 mean=-7.200241088867188e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0194091796875 mean=-0.0005035400390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0341796875 mean=-0.00013828277587890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0206298828125 mean=-0.00104522705078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0380859375 mean=-0.00013446807861328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.023681640625 mean=-0.0007171630859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.039794921875 mean=-2.586841583251953e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02392578125 mean=-0.0010528564453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0400390625 mean=0.00018215179443359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0302734375 mean=-0.00139617919921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037109375 mean=9.632110595703125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.052734375 mean=-0.0023956298828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=7.724761962890625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04052734375 mean=-0.001953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.042724609375 mean=-0.0001392364501953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04345703125 mean=0.00250244140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.87s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1760-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 72.34s +Conversion finished in 448.13709783554077 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1760-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1760-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_daemon_1760/mcore2hf-config.uSVJKk ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1840.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1840.log new file mode 100644 index 0000000000000000000000000000000000000000..d36dddb15e0a008a91f493c9f5359f66acfc1f11 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1840.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_daemon_1840 ++ mkdir -p /tmp/mcore2hf_daemon_1840 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 1840 ']' ++ MANUAL_ITER=1840 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840 ++ CHECKPOINT_BASENAME=load_view1840 ++ [[ load_view1840 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_daemon_1840/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_daemon_1840/mcore2hf-config.tn9bue ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_daemon_1840/mcore2hf-config.tn9bue:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_daemon_1840/mcore2hf-config.tn9bue:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840 +W1016 09:37:05.837000 527783 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1016 09:37:05.837000 527783 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=524288 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=524288 +++ MAX_POSITION_EMBEDDINGS=524288 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840 +W1016 09:37:20.813000 528194 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1016 09:37:20.813000 528194 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 524288 ']' ++ '[' 524288 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_daemon_1840/mcore2hf-config.tn9bue +W1016 09:37:33.683000 528485 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1016 09:37:33.683000 528485 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_daemon_1840/mcore2hf-config.tn9bue/config.json' +✅config.json: /tmp/mcore2hf_daemon_1840/mcore2hf-config.tn9bue/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_daemon_1840/mcore2hf-config.tn9bue/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 524288, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_daemon_1840/mcore2hf-config.tn9bue ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840 1840 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840 ++ local ITER_NUM=1840 ++ '[' -z 1840 ']' +++ printf %07d 1840 ++ local ITER_BASENAME=iter_0001840 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840/iter_0001840 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840/iter_0001840 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840/latest_checkpointed_iteration.txt ++ LATEST_VALUE=1840 ++ '[' 1840 = 1840 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=1840 ++ echo -e '\033[32mLatest checkpoint iteration: 1840\033[0m\n' +Latest checkpoint iteration: 1840 + ++ I=1840 ++ [[ 1840 =~ ^[0-9]+$ ]] +++ printf %07d 1840 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840/iter_0001840 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840/iter_0001840/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840/iter_1840-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1840-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1840-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1840-hf true false bf16 /tmp/mcore2hf_daemon_1840/mcore2hf-config.tn9bue 1840 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1840-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 48757 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_daemon_1840/mcore2hf-config.tn9bue --hf-dir /tmp/mcore2hf_daemon_1840/mcore2hf-config.tn9bue --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 524288 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1840-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 1840 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1016 09:37:53.658000 529055 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1016 09:37:53.658000 529055 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 1840 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_daemon_1840/mcore2hf-config.tn9bue + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 524288 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1840-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_daemon_1840/mcore2hf-config.tn9bue + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.132 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_tn9bue/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_tn9bue/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_tn9bue.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840 at iteration 1840 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1840 [ t 1/1, p 1/1 ] at iteration 1840 +[convert] synchronizer created in 390.48s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07666015625 mean=-0.00048828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10107421875 mean=0.00146484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.078125 mean=-0.00012493133544921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.103515625 mean=0.00010776519775390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.083984375 mean=-0.00020599365234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.078125 mean=0.00103759765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08740234375 mean=-0.000278472900390625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07470703125 mean=0.0015716552734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.095703125 mean=-0.00023365020751953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.072265625 mean=-0.0007781982421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0908203125 mean=-0.0001049041748046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06103515625 mean=-0.0001926422119140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09326171875 mean=-0.00010919570922851562 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.060791015625 mean=0.00011968612670898438 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.087890625 mean=3.8623809814453125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.054931640625 mean=-0.0004119873046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.083984375 mean=5.4836273193359375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.052734375 mean=-0.000476837158203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08203125 mean=-2.3245811462402344e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.064453125 mean=-0.000698089599609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.00019359588623046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.052734375 mean=-0.0003185272216796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07177734375 mean=7.677078247070312e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0419921875 mean=-0.000751495361328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.00013446807861328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.044921875 mean=-0.0008392333984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07373046875 mean=0.00022220611572265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.047607421875 mean=-0.0003681182861328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.072265625 mean=0.0002918243408203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04541015625 mean=-0.0004787445068359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06982421875 mean=0.0003223419189453125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04443359375 mean=-0.00151824951171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06591796875 mean=0.0002651214599609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.044189453125 mean=-0.00189971923828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06884765625 mean=0.000553131103515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.048095703125 mean=-0.00168609619140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06787109375 mean=0.0003490447998046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.048828125 mean=-0.00115203857421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060546875 mean=0.0003452301025390625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.039306640625 mean=-0.0019683837890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.055908203125 mean=0.000286102294921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.041748046875 mean=-0.0028839111328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.053466796875 mean=0.0004215240478515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.035400390625 mean=-0.0020904541015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052490234375 mean=0.0004177093505859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03369140625 mean=-0.0022735595703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=0.0003814697265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03173828125 mean=-0.0014801025390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=0.0003032684326171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.029541015625 mean=-0.00146484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.051513671875 mean=0.0003910064697265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02880859375 mean=-0.00146484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.047607421875 mean=0.000362396240234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02587890625 mean=-0.00067138671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04833984375 mean=0.000514984130859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0234375 mean=-0.00074005126953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.0008087158203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0240478515625 mean=-0.000896453857421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=0.00054168701171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.025390625 mean=-0.0003528594970703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04638671875 mean=0.000141143798828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.024658203125 mean=-0.00021076202392578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=7.677078247070312e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.027099609375 mean=7.343292236328125e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.0001316070556640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.026123046875 mean=0.000209808349609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.056884765625 mean=3.8623809814453125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.031494140625 mean=0.0004558563232421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.058837890625 mean=3.266334533691406e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042236328125 mean=0.000911712646484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05859375 mean=7.963180541992188e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04248046875 mean=0.0012664794921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.057373046875 mean=9.822845458984375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03662109375 mean=0.001739501953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0556640625 mean=5.91278076171875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03662109375 mean=0.00135040283203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052001953125 mean=8.916854858398438e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.025146484375 mean=0.000583648681640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048828125 mean=8.630752563476562e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0234375 mean=0.000244140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0439453125 mean=0.00030517578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0201416015625 mean=-0.0002765655517578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0380859375 mean=0.00016117095947265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0191650390625 mean=-0.0001678466796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.035400390625 mean=0.000446319580078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0211181640625 mean=-0.0004405975341796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03564453125 mean=9.489059448242188e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02294921875 mean=-0.0008392333984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.035888671875 mean=0.0002613067626953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0201416015625 mean=-0.000530242919921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0361328125 mean=0.00019073486328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0230712890625 mean=-0.000843048095703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03369140625 mean=-7.343292236328125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.019287109375 mean=-0.000507354736328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0341796875 mean=-0.0001392364501953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0205078125 mean=-0.000942230224609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037841796875 mean=-0.000133514404296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.023681640625 mean=-0.000682830810546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.039794921875 mean=-2.968311309814453e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0238037109375 mean=-0.00116729736328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0400390625 mean=0.00017833709716796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0303955078125 mean=-0.00140380859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037109375 mean=0.00010633468627929688 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.052734375 mean=-0.0022735595703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=7.486343383789062e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.040283203125 mean=-0.0020751953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.042724609375 mean=-0.00014400482177734375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043212890625 mean=0.00250244140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.93s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1840-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 77.44s +Conversion finished in 470.28230118751526 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1840-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1840-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_daemon_1840/mcore2hf-config.tn9bue ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1920.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1920.log new file mode 100644 index 0000000000000000000000000000000000000000..92a2a3c37d7bd7d8dcb9ddbb040a8a51ae0d10e9 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1920.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_daemon_1920 ++ mkdir -p /tmp/mcore2hf_daemon_1920 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 1920 ']' ++ MANUAL_ITER=1920 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920 ++ CHECKPOINT_BASENAME=load_view1920 ++ [[ load_view1920 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_daemon_1920/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_daemon_1920/mcore2hf-config.XG2N24 ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_daemon_1920/mcore2hf-config.XG2N24:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_daemon_1920/mcore2hf-config.XG2N24:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920 +W1016 14:37:46.473000 939847 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1016 14:37:46.473000 939847 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=524288 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=524288 +++ MAX_POSITION_EMBEDDINGS=524288 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920 +W1016 14:38:03.227000 940529 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1016 14:38:03.227000 940529 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 524288 ']' ++ '[' 524288 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_daemon_1920/mcore2hf-config.XG2N24 +W1016 14:38:16.186000 940848 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1016 14:38:16.186000 940848 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_daemon_1920/mcore2hf-config.XG2N24/config.json' +✅config.json: /tmp/mcore2hf_daemon_1920/mcore2hf-config.XG2N24/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_daemon_1920/mcore2hf-config.XG2N24/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 524288, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_daemon_1920/mcore2hf-config.XG2N24 ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920 1920 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920 ++ local ITER_NUM=1920 ++ '[' -z 1920 ']' +++ printf %07d 1920 ++ local ITER_BASENAME=iter_0001920 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920/iter_0001920 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920/iter_0001920 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920/latest_checkpointed_iteration.txt ++ LATEST_VALUE=1920 ++ '[' 1920 = 1920 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=1920 ++ echo -e '\033[32mLatest checkpoint iteration: 1920\033[0m\n' +Latest checkpoint iteration: 1920 + ++ I=1920 ++ [[ 1920 =~ ^[0-9]+$ ]] +++ printf %07d 1920 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920/iter_0001920 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920/iter_0001920/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920/iter_1920-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1920-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1920-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1920-hf true false bf16 /tmp/mcore2hf_daemon_1920/mcore2hf-config.XG2N24 1920 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1920-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 47167 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_daemon_1920/mcore2hf-config.XG2N24 --hf-dir /tmp/mcore2hf_daemon_1920/mcore2hf-config.XG2N24 --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 524288 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1920-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 1920 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1016 14:38:36.222000 941250 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1016 14:38:36.222000 941250 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 1920 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_daemon_1920/mcore2hf-config.XG2N24 + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 524288 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1920-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_daemon_1920/mcore2hf-config.XG2N24 + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.133 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_XG2N24/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_XG2N24/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_XG2N24.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920 at iteration 1920 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1920 [ t 1/1, p 1/1 ] at iteration 1920 +[convert] synchronizer created in 387.43s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07666015625 mean=-0.000484466552734375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.1005859375 mean=0.0013885498046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.078125 mean=-0.00012493133544921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.103515625 mean=9.441375732421875e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.083984375 mean=-0.000202178955078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07763671875 mean=0.001007080078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08740234375 mean=-0.0002727508544921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07470703125 mean=0.00156402587890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.095703125 mean=-0.00022983551025390625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07177734375 mean=-0.000885009765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0908203125 mean=-0.0001068115234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06103515625 mean=-0.0003337860107421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09326171875 mean=-0.00011157989501953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.060546875 mean=4.410743713378906e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.087890625 mean=4.291534423828125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.054443359375 mean=-0.000499725341796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.083984375 mean=5.626678466796875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.052490234375 mean=-0.000499725341796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08203125 mean=-2.8133392333984375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06396484375 mean=-0.00067901611328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.00019359588623046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.052490234375 mean=-0.0002498626708984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07177734375 mean=7.200241088867188e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0419921875 mean=-0.000743865966796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.000133514404296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.044921875 mean=-0.00069427490234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07373046875 mean=0.00022029876708984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.047119140625 mean=-0.0004367828369140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07275390625 mean=0.0002899169921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.045166015625 mean=-0.0004329681396484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06982421875 mean=0.0003223419189453125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0439453125 mean=-0.00142669677734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06591796875 mean=0.00026702880859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043701171875 mean=-0.00180816650390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0693359375 mean=0.00055694580078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.047607421875 mean=-0.0016632080078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06787109375 mean=0.0003528594970703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04833984375 mean=-0.00116729736328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060791015625 mean=0.0003452301025390625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0390625 mean=-0.002105712890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.055908203125 mean=0.0002803802490234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04150390625 mean=-0.0028839111328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.053466796875 mean=0.0004215240478515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03515625 mean=-0.0020904541015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052490234375 mean=0.0004138946533203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.033203125 mean=-0.0022125244140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=0.0003814697265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.031494140625 mean=-0.001434326171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=0.0003032684326171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.029296875 mean=-0.00130462646484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.051513671875 mean=0.000385284423828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0286865234375 mean=-0.00139617919921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.047607421875 mean=0.000362396240234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0257568359375 mean=-0.00066375732421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048583984375 mean=0.00051116943359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.023193359375 mean=-0.0007476806640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.00080108642578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0238037109375 mean=-0.000911712646484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=0.000537872314453125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.025146484375 mean=-0.0003032684326171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04638671875 mean=0.000141143798828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0245361328125 mean=-0.00013256072998046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=7.82012939453125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02685546875 mean=0.00012969970703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.00012969970703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0260009765625 mean=0.0002841949462890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.056884765625 mean=3.7670135498046875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.031494140625 mean=0.000396728515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.058837890625 mean=3.2901763916015625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042236328125 mean=0.00115966796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05859375 mean=8.058547973632812e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042236328125 mean=0.00141143798828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.057373046875 mean=9.822845458984375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03662109375 mean=0.001617431640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0556640625 mean=5.9604644775390625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03662109375 mean=0.0013275146484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052001953125 mean=8.96453857421875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0250244140625 mean=0.000621795654296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048828125 mean=8.821487426757812e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0234375 mean=0.000400543212890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0439453125 mean=0.00030517578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02001953125 mean=-0.000331878662109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=0.000152587890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.01904296875 mean=-0.00023555755615234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03564453125 mean=0.000446319580078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0208740234375 mean=-0.000392913818359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.035400390625 mean=9.489059448242188e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.022705078125 mean=-0.000850677490234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.035888671875 mean=0.0002593994140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02001953125 mean=-0.000530242919921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0361328125 mean=0.00019073486328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02294921875 mean=-0.00077056884765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03369140625 mean=-7.62939453125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.019287109375 mean=-0.000553131103515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0341796875 mean=-0.0001430511474609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0206298828125 mean=-0.000820159912109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037841796875 mean=-0.0001316070556640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0235595703125 mean=-0.000705718994140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.039794921875 mean=-2.6345252990722656e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.023681640625 mean=-0.00115203857421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.039794921875 mean=0.00018310546875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0301513671875 mean=-0.00147247314453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037109375 mean=0.00010967254638671875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.052490234375 mean=-0.0023040771484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=7.05718994140625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.040283203125 mean=-0.0021209716796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.042724609375 mean=-0.000148773193359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04296875 mean=0.0025787353515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.94s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1920-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 77.22s +Conversion finished in 466.9749581813812 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1920-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1920-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_daemon_1920/mcore2hf-config.XG2N24 ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter2000.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter2000.log new file mode 100644 index 0000000000000000000000000000000000000000..927c6a8a2e6da3591213f55658266a01aa633e2e --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter2000.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_daemon_2000 ++ mkdir -p /tmp/mcore2hf_daemon_2000 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 2000 ']' ++ MANUAL_ITER=2000 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000 ++ CHECKPOINT_BASENAME=load_view2000 ++ [[ load_view2000 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_daemon_2000/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_daemon_2000/mcore2hf-config.wOQF18 ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_daemon_2000/mcore2hf-config.wOQF18:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_daemon_2000/mcore2hf-config.wOQF18:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000 +W1016 19:39:24.194000 292473 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1016 19:39:24.194000 292473 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=524288 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=524288 +++ MAX_POSITION_EMBEDDINGS=524288 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000 +W1016 19:39:39.200000 292838 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1016 19:39:39.200000 292838 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 524288 ']' ++ '[' 524288 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_daemon_2000/mcore2hf-config.wOQF18 +W1016 19:39:51.840000 293106 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1016 19:39:51.840000 293106 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_daemon_2000/mcore2hf-config.wOQF18/config.json' +✅config.json: /tmp/mcore2hf_daemon_2000/mcore2hf-config.wOQF18/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_daemon_2000/mcore2hf-config.wOQF18/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 524288, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_daemon_2000/mcore2hf-config.wOQF18 ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000 2000 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000 ++ local ITER_NUM=2000 ++ '[' -z 2000 ']' +++ printf %07d 2000 ++ local ITER_BASENAME=iter_0002000 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000/iter_0002000 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000/iter_0002000 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000/latest_checkpointed_iteration.txt ++ LATEST_VALUE=2000 ++ '[' 2000 = 2000 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=2000 ++ echo -e '\033[32mLatest checkpoint iteration: 2000\033[0m\n' +Latest checkpoint iteration: 2000 + ++ I=2000 ++ [[ 2000 =~ ^[0-9]+$ ]] +++ printf %07d 2000 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000/iter_0002000 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000/iter_0002000/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000/iter_2000-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2000-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2000-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2000-hf true false bf16 /tmp/mcore2hf_daemon_2000/mcore2hf-config.wOQF18 2000 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2000-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 53587 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_daemon_2000/mcore2hf-config.wOQF18 --hf-dir /tmp/mcore2hf_daemon_2000/mcore2hf-config.wOQF18 --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 524288 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2000-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 2000 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1016 19:40:11.841000 293651 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1016 19:40:11.841000 293651 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 2000 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_daemon_2000/mcore2hf-config.wOQF18 + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 524288 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2000-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_daemon_2000/mcore2hf-config.wOQF18 + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.129 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_wOQF18/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_wOQF18/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_wOQF18.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000 at iteration 2000 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2000 [ t 1/1, p 1/1 ] at iteration 2000 +[convert] synchronizer created in 403.18s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07666015625 mean=-0.0004787445068359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.1005859375 mean=0.001495361328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.078125 mean=-0.000118255615234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10302734375 mean=0.00015354156494140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.083984375 mean=-0.00020313262939453125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0771484375 mean=0.00101470947265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08740234375 mean=-0.000278472900390625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07421875 mean=0.00150299072265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.095703125 mean=-0.00022602081298828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07177734375 mean=-0.000827789306640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0908203125 mean=-0.00010824203491210938 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.060791015625 mean=-0.000370025634765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09326171875 mean=-0.00011157989501953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.060302734375 mean=8.58306884765625e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08740234375 mean=4.38690185546875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.054443359375 mean=-0.000484466552734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.083984375 mean=5.793571472167969e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05224609375 mean=-0.0004520416259765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08203125 mean=-2.4080276489257812e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06396484375 mean=-0.000701904296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.00019741058349609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05224609375 mean=-0.0002346038818359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07177734375 mean=7.2479248046875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0419921875 mean=-0.0008392333984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.000133514404296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.044677734375 mean=-0.0008544921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07373046875 mean=0.00022029876708984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.047119140625 mean=-0.0004825592041015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07275390625 mean=0.0002880096435546875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.044921875 mean=-0.000347137451171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06982421875 mean=0.0003261566162109375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043701171875 mean=-0.00131988525390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06591796875 mean=0.0002651214599609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04345703125 mean=-0.0017547607421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0693359375 mean=0.000560760498046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.047607421875 mean=-0.0015869140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06787109375 mean=0.0003528594970703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.048095703125 mean=-0.00122833251953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060791015625 mean=0.000347137451171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03857421875 mean=-0.0020599365234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.055908203125 mean=0.00028228759765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.041259765625 mean=-0.0028533935546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0537109375 mean=0.0004177093505859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.034912109375 mean=-0.00213623046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052490234375 mean=0.0004119873046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.032958984375 mean=-0.00213623046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=0.0003795623779296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.031494140625 mean=-0.001556396484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=0.0003070831298828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.029052734375 mean=-0.00125885009765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.0003814697265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.028564453125 mean=-0.0014495849609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.047607421875 mean=0.0003681182861328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.025634765625 mean=-0.000705718994140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048583984375 mean=0.000507354736328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0230712890625 mean=-0.000751495361328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052001953125 mean=0.000812530517578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0238037109375 mean=-0.000850677490234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=0.000537872314453125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0250244140625 mean=-0.0003490447998046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04638671875 mean=0.0001373291015625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.024658203125 mean=-7.62939453125e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=7.43865966796875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02685546875 mean=5.364418029785156e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.00013065338134765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0260009765625 mean=0.000362396240234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.056884765625 mean=3.838539123535156e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03125 mean=0.0004520416259765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.058837890625 mean=3.24249267578125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0419921875 mean=0.0010833740234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.058349609375 mean=8.106231689453125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04248046875 mean=0.00145721435546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.057373046875 mean=9.918212890625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.036376953125 mean=0.001617431640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0556640625 mean=5.841255187988281e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.036376953125 mean=0.001312255859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052001953125 mean=8.869171142578125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.025146484375 mean=0.00054931640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048828125 mean=8.916854858398438e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0233154296875 mean=0.0003681182861328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0439453125 mean=0.0002994537353515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.019775390625 mean=-0.0003871917724609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0380859375 mean=0.0001506805419921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.01904296875 mean=-0.00016117095947265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03564453125 mean=0.0004329681396484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0206298828125 mean=-0.00028228759765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.035400390625 mean=9.1552734375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0225830078125 mean=-0.000812530517578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.035888671875 mean=0.000255584716796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0201416015625 mean=-0.0004444122314453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036376953125 mean=0.00018787384033203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0228271484375 mean=-0.00074005126953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03369140625 mean=-7.43865966796875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.019287109375 mean=-0.00057220458984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0341796875 mean=-0.00014972686767578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0205078125 mean=-0.00084686279296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037841796875 mean=-0.00013446807861328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0233154296875 mean=-0.000732421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.039794921875 mean=-2.2411346435546875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0234375 mean=-0.00121307373046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.039794921875 mean=0.000186920166015625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0299072265625 mean=-0.0014801025390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037109375 mean=0.00011730194091796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.052490234375 mean=-0.0022735595703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=6.580352783203125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0400390625 mean=-0.0021514892578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.042724609375 mean=-0.0001468658447265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04296875 mean=0.002655029296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.97s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2000-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 78.49s +Conversion finished in 483.96730494499207 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2000-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2000-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_daemon_2000/mcore2hf-config.wOQF18 ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter2240.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter2240.log new file mode 100644 index 0000000000000000000000000000000000000000..c8c475349734357a5a17e34cead27abc8117f39f --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter2240.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_daemon_2240 ++ mkdir -p /tmp/mcore2hf_daemon_2240 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 2240 ']' ++ MANUAL_ITER=2240 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240 ++ CHECKPOINT_BASENAME=load_view2240 ++ [[ load_view2240 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_daemon_2240/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_daemon_2240/mcore2hf-config.WzCIji ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_daemon_2240/mcore2hf-config.WzCIji:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_daemon_2240/mcore2hf-config.WzCIji:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240 +W1017 10:42:36.170000 263395 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1017 10:42:36.170000 263395 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=524288 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=524288 +++ MAX_POSITION_EMBEDDINGS=524288 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240 +W1017 10:42:50.945000 264005 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1017 10:42:50.945000 264005 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 524288 ']' ++ '[' 524288 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_daemon_2240/mcore2hf-config.WzCIji +W1017 10:43:03.684000 264269 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1017 10:43:03.684000 264269 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_daemon_2240/mcore2hf-config.WzCIji/config.json' +✅config.json: /tmp/mcore2hf_daemon_2240/mcore2hf-config.WzCIji/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_daemon_2240/mcore2hf-config.WzCIji/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 524288, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_daemon_2240/mcore2hf-config.WzCIji ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240 2240 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240 ++ local ITER_NUM=2240 ++ '[' -z 2240 ']' +++ printf %07d 2240 ++ local ITER_BASENAME=iter_0002240 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240/iter_0002240 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240/iter_0002240 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240/latest_checkpointed_iteration.txt ++ LATEST_VALUE=2240 ++ '[' 2240 = 2240 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=2240 ++ echo -e '\033[32mLatest checkpoint iteration: 2240\033[0m\n' +Latest checkpoint iteration: 2240 + ++ I=2240 ++ [[ 2240 =~ ^[0-9]+$ ]] +++ printf %07d 2240 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240/iter_0002240 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240/iter_0002240/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240/iter_2240-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2240-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2240-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2240-hf true false bf16 /tmp/mcore2hf_daemon_2240/mcore2hf-config.WzCIji 2240 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2240-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 38843 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_daemon_2240/mcore2hf-config.WzCIji --hf-dir /tmp/mcore2hf_daemon_2240/mcore2hf-config.WzCIji --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 524288 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2240-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 2240 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1017 10:43:23.498000 264731 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1017 10:43:23.498000 264731 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 2240 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_daemon_2240/mcore2hf-config.WzCIji + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 524288 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2240-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_daemon_2240/mcore2hf-config.WzCIji + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.128 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_WzCIji/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_WzCIji/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_WzCIji.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240 at iteration 2240 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2240 [ t 1/1, p 1/1 ] at iteration 2240 +[convert] synchronizer created in 387.83s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07666015625 mean=-0.00048065185546875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.099609375 mean=0.00147247314453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.078125 mean=-0.0001163482666015625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10205078125 mean=0.00032806396484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.083984375 mean=-0.0001983642578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07666015625 mean=0.00064849853515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08740234375 mean=-0.0002918243408203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0732421875 mean=0.00124359130859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.095703125 mean=-0.0002288818359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07080078125 mean=-0.000545501708984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0908203125 mean=-0.0001163482666015625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.060302734375 mean=-0.0003662109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0927734375 mean=-0.00012493133544921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.059326171875 mean=0.00010967254638671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.087890625 mean=4.863739013671875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.053955078125 mean=-0.0003662109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.083984375 mean=5.650520324707031e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05224609375 mean=-0.00020694732666015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08203125 mean=-2.9087066650390625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06298828125 mean=-0.00069427490234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.0001926422119140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.051025390625 mean=-0.0003643035888671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.072265625 mean=5.888938903808594e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.041259765625 mean=-0.000823974609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.00012683868408203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0439453125 mean=-0.000743865966796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07421875 mean=0.0002193450927734375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.046142578125 mean=-0.000514984130859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07275390625 mean=0.000286102294921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.044189453125 mean=-0.000385284423828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06982421875 mean=0.00032806396484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042724609375 mean=-0.0013275146484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06640625 mean=0.0002689361572265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042724609375 mean=-0.00177764892578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0693359375 mean=0.0005645751953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.046875 mean=-0.00156402587890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.068359375 mean=0.000354766845703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04736328125 mean=-0.00098419189453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06103515625 mean=0.000347137451171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03759765625 mean=-0.0017852783203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05615234375 mean=0.000278472900390625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04052734375 mean=-0.002777099609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0537109375 mean=0.0004177093505859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0341796875 mean=-0.0020294189453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052490234375 mean=0.0004138946533203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0322265625 mean=-0.00213623046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.050048828125 mean=0.0003719329833984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0308837890625 mean=-0.00153350830078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04931640625 mean=0.0003032684326171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0284423828125 mean=-0.00118255615234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.000370025634765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0281982421875 mean=-0.001434326171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0478515625 mean=0.000370025634765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0250244140625 mean=-0.000644683837890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048828125 mean=0.000499725341796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.022705078125 mean=-0.000667572021484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052001953125 mean=0.0008087158203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0234375 mean=-0.00067138671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04931640625 mean=0.000530242919921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.024658203125 mean=-0.0002593994140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.046630859375 mean=0.00014019012451171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0242919921875 mean=-9.775161743164062e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=8.0108642578125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0264892578125 mean=0.0001811981201171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.0001316070556640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.025634765625 mean=0.0001888275146484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.056884765625 mean=3.838539123535156e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.031005859375 mean=0.000457763671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.058837890625 mean=3.743171691894531e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.041748046875 mean=0.0010223388671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.058349609375 mean=7.772445678710938e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042236328125 mean=0.0015411376953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.057373046875 mean=9.870529174804688e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.036376953125 mean=0.00170135498046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0556640625 mean=5.3882598876953125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.036376953125 mean=0.00147247314453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052001953125 mean=8.7738037109375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0247802734375 mean=0.000576019287109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048828125 mean=8.821487426757812e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0230712890625 mean=0.00035858154296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0439453125 mean=0.00029754638671875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0194091796875 mean=-0.000331878662109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=0.000148773193359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0186767578125 mean=-0.00016498565673828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03564453125 mean=0.0004405975341796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0205078125 mean=-0.000255584716796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03564453125 mean=8.916854858398438e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.022216796875 mean=-0.00078582763671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.035888671875 mean=0.000247955322265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0196533203125 mean=-0.00049591064453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036376953125 mean=0.00018405914306640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0225830078125 mean=-0.00078582763671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03369140625 mean=-7.009506225585938e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.01904296875 mean=-0.000537872314453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0341796875 mean=-0.000152587890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02001953125 mean=-0.0009765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03759765625 mean=-0.00014400482177734375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02294921875 mean=-0.000782012939453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.039794921875 mean=-2.4199485778808594e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.023193359375 mean=-0.00119781494140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.039794921875 mean=0.00017547607421875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.029541015625 mean=-0.001312255859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037353515625 mean=0.00010967254638671875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05224609375 mean=-0.0023040771484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=7.05718994140625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.039794921875 mean=-0.002166748046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04296875 mean=-0.00015354156494140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042724609375 mean=0.0025482177734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.95s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2240-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 74.75s +Conversion finished in 464.994833946228 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2240-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2240-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_daemon_2240/mcore2hf-config.WzCIji ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter2320.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter2320.log new file mode 100644 index 0000000000000000000000000000000000000000..db768c51a26e94ae2b8f10887fc9e14e160f95ea --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter2320.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_daemon_2320 ++ mkdir -p /tmp/mcore2hf_daemon_2320 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 2320 ']' ++ MANUAL_ITER=2320 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320 ++ CHECKPOINT_BASENAME=load_view2320 ++ [[ load_view2320 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_daemon_2320/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_daemon_2320/mcore2hf-config.g1FWoT ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_daemon_2320/mcore2hf-config.g1FWoT:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_daemon_2320/mcore2hf-config.g1FWoT:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320 +W1017 15:43:01.826000 632443 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1017 15:43:01.826000 632443 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=524288 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=524288 +++ MAX_POSITION_EMBEDDINGS=524288 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320 +W1017 15:43:16.250000 632950 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1017 15:43:16.250000 632950 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 524288 ']' ++ '[' 524288 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_daemon_2320/mcore2hf-config.g1FWoT +W1017 15:43:28.798000 633163 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1017 15:43:28.798000 633163 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_daemon_2320/mcore2hf-config.g1FWoT/config.json' +✅config.json: /tmp/mcore2hf_daemon_2320/mcore2hf-config.g1FWoT/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_daemon_2320/mcore2hf-config.g1FWoT/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 524288, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_daemon_2320/mcore2hf-config.g1FWoT ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320 2320 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320 ++ local ITER_NUM=2320 ++ '[' -z 2320 ']' +++ printf %07d 2320 ++ local ITER_BASENAME=iter_0002320 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320/iter_0002320 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320/iter_0002320 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320/latest_checkpointed_iteration.txt ++ LATEST_VALUE=2320 ++ '[' 2320 = 2320 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=2320 ++ echo -e '\033[32mLatest checkpoint iteration: 2320\033[0m\n' +Latest checkpoint iteration: 2320 + ++ I=2320 ++ [[ 2320 =~ ^[0-9]+$ ]] +++ printf %07d 2320 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320/iter_0002320 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320/iter_0002320/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320/iter_2320-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2320-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2320-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2320-hf true false bf16 /tmp/mcore2hf_daemon_2320/mcore2hf-config.g1FWoT 2320 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2320-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 53377 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_daemon_2320/mcore2hf-config.g1FWoT --hf-dir /tmp/mcore2hf_daemon_2320/mcore2hf-config.g1FWoT --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 524288 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2320-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 2320 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1017 15:43:48.311000 633626 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1017 15:43:48.311000 633626 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 2320 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_daemon_2320/mcore2hf-config.g1FWoT + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 524288 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2320-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_daemon_2320/mcore2hf-config.g1FWoT + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.133 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.004 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_g1FWoT/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_g1FWoT/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_g1FWoT.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320 at iteration 2320 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2320 [ t 1/1, p 1/1 ] at iteration 2320 +[convert] synchronizer created in 396.41s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07666015625 mean=-0.000476837158203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.099609375 mean=0.00152587890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.078125 mean=-0.00011348724365234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.1015625 mean=0.0003204345703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.083984375 mean=-0.00019359588623046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.076171875 mean=0.000659942626953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08740234375 mean=-0.00029754638671875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07275390625 mean=0.00135040283203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.095703125 mean=-0.00022411346435546875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0703125 mean=-0.00054931640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0908203125 mean=-0.00011301040649414062 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06005859375 mean=-0.0002803802490234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0927734375 mean=-0.00012493133544921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05908203125 mean=0.000270843505859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.087890625 mean=4.935264587402344e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0537109375 mean=-0.000396728515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.083984375 mean=5.626678466796875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0517578125 mean=-0.00018787384033203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08203125 mean=-2.849102020263672e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0625 mean=-0.00066375732421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.00019359588623046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05078125 mean=-0.0002765655517578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.072265625 mean=6.198883056640625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.040771484375 mean=-0.00075531005859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.00012683868408203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043701171875 mean=-0.000759124755859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07421875 mean=0.00021839141845703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0458984375 mean=-0.00058746337890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07275390625 mean=0.000286102294921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0439453125 mean=-0.0003147125244140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06982421875 mean=0.0003299713134765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04248046875 mean=-0.0013885498046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06640625 mean=0.0002727508544921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04248046875 mean=-0.00176239013671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0693359375 mean=0.0005645751953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.046630859375 mean=-0.00154876708984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.068359375 mean=0.0003528594970703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.046875 mean=-0.0009765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06103515625 mean=0.0003490447998046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037353515625 mean=-0.001739501953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05615234375 mean=0.0002803802490234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0400390625 mean=-0.0027618408203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0537109375 mean=0.0004177093505859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.033935546875 mean=-0.0019683837890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052734375 mean=0.0004177093505859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.031982421875 mean=-0.0021209716796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.050048828125 mean=0.0003719329833984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.030517578125 mean=-0.00150299072265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04931640625 mean=0.0003032684326171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0281982421875 mean=-0.001129150390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052001953125 mean=0.00037384033203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02783203125 mean=-0.00122833251953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0478515625 mean=0.0003681182861328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02490234375 mean=-0.000637054443359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048828125 mean=0.000499725341796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0225830078125 mean=-0.000614166259765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052001953125 mean=0.0008087158203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.023193359375 mean=-0.000690460205078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04931640625 mean=0.0005340576171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.024658203125 mean=-0.0002593994140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.046630859375 mean=0.000141143798828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0242919921875 mean=-2.3484230041503906e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.050048828125 mean=8.535385131835938e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0264892578125 mean=0.0001277923583984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.00013446807861328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0255126953125 mean=0.0002593994140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.056884765625 mean=3.695487976074219e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.031005859375 mean=0.000514984130859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.058837890625 mean=3.790855407714844e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.041748046875 mean=0.00107574462890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.058349609375 mean=7.724761962890625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042236328125 mean=0.001434326171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.057373046875 mean=9.775161743164062e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.036376953125 mean=0.0016937255859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.055419921875 mean=5.435943603515625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0361328125 mean=0.00142669677734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052001953125 mean=8.916854858398438e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.024658203125 mean=0.000637054443359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048828125 mean=8.487701416015625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.023193359375 mean=0.0003662109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0439453125 mean=0.00029754638671875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0194091796875 mean=-0.0002536773681640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=0.0001392364501953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0186767578125 mean=-8.487701416015625e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03564453125 mean=0.000438690185546875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.020263671875 mean=-0.000240325927734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03564453125 mean=8.296966552734375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.022216796875 mean=-0.0008392333984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0361328125 mean=0.000247955322265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.01953125 mean=-0.0003337860107421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036376953125 mean=0.0001850128173828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0223388671875 mean=-0.00069427490234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03369140625 mean=-7.200241088867188e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.01904296875 mean=-0.00051116943359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0341796875 mean=-0.000152587890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0198974609375 mean=-0.000957489013671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03759765625 mean=-0.000141143798828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02294921875 mean=-0.00075531005859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.039794921875 mean=-2.2292137145996094e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0230712890625 mean=-0.00115966796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.039794921875 mean=0.0001773834228515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0294189453125 mean=-0.0013275146484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037109375 mean=0.00010538101196289062 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.052001953125 mean=-0.002288818359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=6.818771362304688e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03955078125 mean=-0.002166748046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.042724609375 mean=-0.00015735626220703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04248046875 mean=0.0026397705078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.94s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2320-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 79.92s +Conversion finished in 478.5997612476349 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2320-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2320-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_daemon_2320/mcore2hf-config.g1FWoT ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter2384.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter2384.log new file mode 100644 index 0000000000000000000000000000000000000000..c02063d078f18636690b5f989d3f10c15613e34b --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter2384.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_daemon_2384 ++ mkdir -p /tmp/mcore2hf_daemon_2384 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 2384 ']' ++ MANUAL_ITER=2384 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384 ++ CHECKPOINT_BASENAME=load_view2384 ++ [[ load_view2384 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_daemon_2384/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_daemon_2384/mcore2hf-config.EY81st ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_daemon_2384/mcore2hf-config.EY81st:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_daemon_2384/mcore2hf-config.EY81st:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384 +W1017 19:50:57.850000 942352 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1017 19:50:57.850000 942352 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=524288 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=524288 +++ MAX_POSITION_EMBEDDINGS=524288 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384 +W1017 19:51:11.999000 942797 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1017 19:51:11.999000 942797 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 524288 ']' ++ '[' 524288 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_daemon_2384/mcore2hf-config.EY81st +W1017 19:51:24.590000 943134 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1017 19:51:24.590000 943134 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_daemon_2384/mcore2hf-config.EY81st/config.json' +✅config.json: /tmp/mcore2hf_daemon_2384/mcore2hf-config.EY81st/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_daemon_2384/mcore2hf-config.EY81st/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 524288, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_daemon_2384/mcore2hf-config.EY81st ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384 2384 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384 ++ local ITER_NUM=2384 ++ '[' -z 2384 ']' +++ printf %07d 2384 ++ local ITER_BASENAME=iter_0002384 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384/iter_0002384 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384/iter_0002384 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384/latest_checkpointed_iteration.txt ++ LATEST_VALUE=2384 ++ '[' 2384 = 2384 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=2384 ++ echo -e '\033[32mLatest checkpoint iteration: 2384\033[0m\n' +Latest checkpoint iteration: 2384 + ++ I=2384 ++ [[ 2384 =~ ^[0-9]+$ ]] +++ printf %07d 2384 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384/iter_0002384 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384/iter_0002384/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384/iter_2384-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2384-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2384-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2384-hf true false bf16 /tmp/mcore2hf_daemon_2384/mcore2hf-config.EY81st 2384 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2384-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 60075 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_daemon_2384/mcore2hf-config.EY81st --hf-dir /tmp/mcore2hf_daemon_2384/mcore2hf-config.EY81st --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 524288 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2384-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 2384 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1017 19:51:44.274000 943538 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1017 19:51:44.274000 943538 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 2384 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_daemon_2384/mcore2hf-config.EY81st + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 524288 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2384-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_daemon_2384/mcore2hf-config.EY81st + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.125 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_EY81st/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_EY81st/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_EY81st.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384 at iteration 2384 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2384 [ t 1/1, p 1/1 ] at iteration 2384 +[convert] synchronizer created in 371.65s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07666015625 mean=-0.0004749298095703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.099609375 mean=0.0015869140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.078125 mean=-0.00011396408081054688 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10107421875 mean=0.0002956390380859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.083984375 mean=-0.00018787384033203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.076171875 mean=0.000652313232421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08740234375 mean=-0.00029754638671875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.072265625 mean=0.00136566162109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.095703125 mean=-0.00022411346435546875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0703125 mean=-0.000457763671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0908203125 mean=-0.00011491775512695312 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.059814453125 mean=-0.00030517578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0927734375 mean=-0.000125885009765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.058837890625 mean=0.000194549560546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.087890625 mean=4.839897155761719e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.053466796875 mean=-0.000507354736328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.083984375 mean=5.5789947509765625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.051513671875 mean=-9.918212890625e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08251953125 mean=-2.6106834411621094e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.062255859375 mean=-0.000621795654296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.00019359588623046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05029296875 mean=-0.0003070831298828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.072265625 mean=6.246566772460938e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04052734375 mean=-0.0007781982421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.000125885009765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04345703125 mean=-0.00072479248046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07421875 mean=0.00021457672119140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04541015625 mean=-0.00064849853515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07275390625 mean=0.0002803802490234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043701171875 mean=-0.0002613067626953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06982421875 mean=0.0003261566162109375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042236328125 mean=-0.001373291015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06640625 mean=0.0002727508544921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042236328125 mean=-0.0018157958984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0693359375 mean=0.0005645751953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04638671875 mean=-0.00153350830078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.068359375 mean=0.0003490447998046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.046630859375 mean=-0.00095367431640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06103515625 mean=0.000347137451171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037109375 mean=-0.0017242431640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05615234375 mean=0.0002803802490234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.039794921875 mean=-0.002838134765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0537109375 mean=0.000415802001953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.033935546875 mean=-0.0020751953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052734375 mean=0.0004177093505859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03173828125 mean=-0.0020751953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05029296875 mean=0.0003757476806640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0303955078125 mean=-0.0014801025390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049560546875 mean=0.0003032684326171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0279541015625 mean=-0.001129150390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052001953125 mean=0.0003795623779296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0277099609375 mean=-0.0013275146484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0478515625 mean=0.0003662109375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0247802734375 mean=-0.00061798095703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048828125 mean=0.00049591064453125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0224609375 mean=-0.0005950927734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05224609375 mean=0.0008087158203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0230712890625 mean=-0.00072479248046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04931640625 mean=0.0005340576171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0245361328125 mean=-0.0002574920654296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.046630859375 mean=0.000141143798828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.024169921875 mean=0.00010585784912109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.050048828125 mean=8.535385131835938e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0263671875 mean=3.790855407714844e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.0001354217529296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0255126953125 mean=0.00021839141845703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.056884765625 mean=3.695487976074219e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03076171875 mean=0.000492095947265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.058837890625 mean=3.8623809814453125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.041748046875 mean=0.0010833740234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.058349609375 mean=8.0108642578125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0419921875 mean=0.00144195556640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.057373046875 mean=9.72747802734375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0361328125 mean=0.00168609619140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.055419921875 mean=5.3882598876953125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0361328125 mean=0.001434326171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052001953125 mean=8.96453857421875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.024658203125 mean=0.00055694580078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048828125 mean=8.249282836914062e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.023193359375 mean=0.0003566741943359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0439453125 mean=0.000293731689453125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.019287109375 mean=-0.0002841949462890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=0.00014019012451171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0186767578125 mean=1.049041748046875e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03564453125 mean=0.00043487548828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.020263671875 mean=-0.00022602081298828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03564453125 mean=8.106231689453125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02197265625 mean=-0.000812530517578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.035888671875 mean=0.0002574920654296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.01953125 mean=-0.000408172607421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03662109375 mean=0.0001850128173828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0223388671875 mean=-0.00070953369140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03369140625 mean=-7.104873657226562e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0189208984375 mean=-0.0003490447998046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0341796875 mean=-0.0001544952392578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.019775390625 mean=-0.00086212158203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03759765625 mean=-0.0001392364501953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0228271484375 mean=-0.000743865966796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.039794921875 mean=-2.3365020751953125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02294921875 mean=-0.0011138916015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.039794921875 mean=0.0001735687255859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.029296875 mean=-0.00131988525390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037353515625 mean=0.00010728836059570312 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0517578125 mean=-0.0022125244140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=6.4849853515625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.039306640625 mean=-0.002105712890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.042724609375 mean=-0.00015926361083984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04248046875 mean=0.002593994140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.81s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2384-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 70.69s +Conversion finished in 444.38420820236206 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2384-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2384-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_daemon_2384/mcore2hf-config.EY81st ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter40.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter40.log new file mode 100644 index 0000000000000000000000000000000000000000..490a3704e94b4963cdb2739da28fbede7b480799 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter40.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_daemon_40 ++ mkdir -p /tmp/mcore2hf_daemon_40 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 40 ']' ++ MANUAL_ITER=40 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40 ++ CHECKPOINT_BASENAME=load_view40 ++ [[ load_view40 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_daemon_40/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_daemon_40/mcore2hf-config.OEx9nV ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_daemon_40/mcore2hf-config.OEx9nV:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_daemon_40/mcore2hf-config.OEx9nV:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40 +W1011 16:35:11.352000 781769 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 16:35:11.352000 781769 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=524288 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=524288 +++ MAX_POSITION_EMBEDDINGS=524288 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40 +W1011 16:35:26.922000 782375 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 16:35:26.922000 782375 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 524288 ']' ++ '[' 524288 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_daemon_40/mcore2hf-config.OEx9nV +W1011 16:35:40.162000 782685 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 16:35:40.162000 782685 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_daemon_40/mcore2hf-config.OEx9nV/config.json' +✅config.json: /tmp/mcore2hf_daemon_40/mcore2hf-config.OEx9nV/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_daemon_40/mcore2hf-config.OEx9nV/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 524288, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_daemon_40/mcore2hf-config.OEx9nV ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40 40 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40 ++ local ITER_NUM=40 ++ '[' -z 40 ']' +++ printf %07d 40 ++ local ITER_BASENAME=iter_0000040 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40/iter_0000040 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40/iter_0000040 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40/latest_checkpointed_iteration.txt ++ LATEST_VALUE=40 ++ '[' 40 = 40 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=40 ++ echo -e '\033[32mLatest checkpoint iteration: 40\033[0m\n' +Latest checkpoint iteration: 40 + ++ I=40 ++ [[ 40 =~ ^[0-9]+$ ]] +++ printf %07d 40 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40/iter_0000040 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40/iter_0000040/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40/iter_40-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_40-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_40-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_40-hf true false bf16 /tmp/mcore2hf_daemon_40/mcore2hf-config.OEx9nV 40 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_40-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 48473 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_daemon_40/mcore2hf-config.OEx9nV --hf-dir /tmp/mcore2hf_daemon_40/mcore2hf-config.OEx9nV --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 524288 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_40-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 40 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1011 16:36:00.887000 783135 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 16:36:00.887000 783135 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 40 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_daemon_40/mcore2hf-config.OEx9nV + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 524288 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_40-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_daemon_40/mcore2hf-config.OEx9nV + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.136 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_OEx9nV/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_OEx9nV/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_OEx9nV.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40 at iteration 40 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view40 [ t 1/1, p 1/1 ] at iteration 40 +[convert] synchronizer created in 399.14s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.078125 mean=-0.0005035400390625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10498046875 mean=0.00186920166015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.080078125 mean=-0.000164031982421875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.1083984375 mean=-8.344650268554688e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0869140625 mean=-0.00026702880859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.08056640625 mean=0.000762939453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09033203125 mean=-0.00020503997802734375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07958984375 mean=0.0012969970703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0986328125 mean=-0.00023365020751953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07421875 mean=-0.00087738037109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09375 mean=-7.486343383789062e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0625 mean=-0.00022029876708984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.095703125 mean=-7.82012939453125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06298828125 mean=7.2479248046875e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09033203125 mean=2.872943878173828e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0556640625 mean=-0.00014400482177734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08642578125 mean=2.5033950805664062e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0537109375 mean=-0.0005035400390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.083984375 mean=7.05718994140625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0654296875 mean=-0.0004405975341796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.078125 mean=0.00022125244140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.052734375 mean=-0.00058746337890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07373046875 mean=0.00015354156494140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042724609375 mean=-0.000926971435546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07763671875 mean=0.00016689300537109375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.045654296875 mean=-0.00064849853515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07568359375 mean=0.00023174285888671875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.048583984375 mean=-0.000965118408203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07421875 mean=0.000293731689453125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.047607421875 mean=-0.000762939453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0712890625 mean=0.0002994537353515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.046630859375 mean=-0.00130462646484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0673828125 mean=0.0002841949462890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.047119140625 mean=-0.0022430419921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0703125 mean=0.0006103515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04931640625 mean=-0.00157928466796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06884765625 mean=0.0003871917724609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.048828125 mean=-0.0010528564453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.061767578125 mean=0.000415802001953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.040771484375 mean=-0.0019989013671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.056640625 mean=0.000354766845703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043701171875 mean=-0.0029449462890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.054931640625 mean=0.00048065185546875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.036865234375 mean=-0.0021209716796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.053955078125 mean=0.000438690185546875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.034912109375 mean=-0.00238037109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.051025390625 mean=0.0004062652587890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.033203125 mean=-0.00171661376953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.050537109375 mean=0.00032806396484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0308837890625 mean=-0.00174713134765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052978515625 mean=0.000457763671875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.029052734375 mean=-0.0017242431640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04931640625 mean=0.000408172607421875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0267333984375 mean=-0.000888824462890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=0.00054931640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0233154296875 mean=-0.001007080078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052734375 mean=0.00084686279296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0245361328125 mean=-0.000885009765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.050537109375 mean=0.000530242919921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0255126953125 mean=-0.000583648681640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.047607421875 mean=0.00013065338134765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0247802734375 mean=-0.00011873245239257812 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05126953125 mean=9.72747802734375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.027099609375 mean=-4.744529724121094e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.053466796875 mean=0.0001850128173828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0267333984375 mean=0.00015544891357421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.058837890625 mean=8.0108642578125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03173828125 mean=0.0003814697265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060791015625 mean=5.364418029785156e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04248046875 mean=0.0009765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060546875 mean=0.00011205673217773438 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042724609375 mean=0.00131988525390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05908203125 mean=0.0001239776611328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037109375 mean=0.00157928466796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0576171875 mean=8.440017700195312e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.036376953125 mean=0.00133514404296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05419921875 mean=0.00013256072998046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0255126953125 mean=0.0003261566162109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05078125 mean=0.0001373291015625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0240478515625 mean=0.00020313262939453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.045654296875 mean=0.00037384033203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0201416015625 mean=-0.0005645751953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.039794921875 mean=0.0001983642578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0198974609375 mean=-0.000377655029296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036865234375 mean=0.0004253387451171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0208740234375 mean=-0.00035858154296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037109375 mean=0.0001316070556640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0230712890625 mean=-0.000835418701171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03759765625 mean=0.00035858154296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0203857421875 mean=-0.00054168701171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03759765625 mean=0.00021648406982421875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02294921875 mean=-0.00078582763671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03515625 mean=-7.963180541992188e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0189208984375 mean=-0.00055694580078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.034912109375 mean=-0.00010395050048828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02099609375 mean=-0.000957489013671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0380859375 mean=-0.00010967254638671875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0224609375 mean=-0.000797271728515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=1.9073486328125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0224609375 mean=-0.00101470947265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04052734375 mean=0.00021266937255859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0289306640625 mean=-0.00133514404296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038818359375 mean=6.532669067382812e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0498046875 mean=-0.0022430419921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0400390625 mean=6.67572021484375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037841796875 mean=-0.00183868408203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04443359375 mean=-0.000152587890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.041259765625 mean=0.0024261474609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.95s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_40-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 79.48s +Conversion finished in 480.8764617443085 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_40-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_40-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_daemon_40/mcore2hf-config.OEx9nV ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter400.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter400.log new file mode 100644 index 0000000000000000000000000000000000000000..f342124d2764c399927ed36bdd61f49ebc24a00a --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter400.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_daemon_400 ++ mkdir -p /tmp/mcore2hf_daemon_400 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 400 ']' ++ MANUAL_ITER=400 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400 ++ CHECKPOINT_BASENAME=load_view400 ++ [[ load_view400 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_daemon_400/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_daemon_400/mcore2hf-config.vkfwHT ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_daemon_400/mcore2hf-config.vkfwHT:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_daemon_400/mcore2hf-config.vkfwHT:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400 +W1012 15:03:41.050000 260652 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1012 15:03:41.050000 260652 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=524288 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=524288 +++ MAX_POSITION_EMBEDDINGS=524288 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400 +W1012 15:03:56.460000 261655 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1012 15:03:56.460000 261655 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 524288 ']' ++ '[' 524288 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_daemon_400/mcore2hf-config.vkfwHT +W1012 15:04:09.757000 261962 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1012 15:04:09.757000 261962 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_daemon_400/mcore2hf-config.vkfwHT/config.json' +✅config.json: /tmp/mcore2hf_daemon_400/mcore2hf-config.vkfwHT/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_daemon_400/mcore2hf-config.vkfwHT/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 524288, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_daemon_400/mcore2hf-config.vkfwHT ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400 400 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400 ++ local ITER_NUM=400 ++ '[' -z 400 ']' +++ printf %07d 400 ++ local ITER_BASENAME=iter_0000400 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400/iter_0000400 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400/iter_0000400 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400/latest_checkpointed_iteration.txt ++ LATEST_VALUE=400 ++ '[' 400 = 400 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=400 ++ echo -e '\033[32mLatest checkpoint iteration: 400\033[0m\n' +Latest checkpoint iteration: 400 + ++ I=400 ++ [[ 400 =~ ^[0-9]+$ ]] +++ printf %07d 400 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400/iter_0000400 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400/iter_0000400/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400/iter_400-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_400-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_400-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_400-hf true false bf16 /tmp/mcore2hf_daemon_400/mcore2hf-config.vkfwHT 400 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_400-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 41603 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_daemon_400/mcore2hf-config.vkfwHT --hf-dir /tmp/mcore2hf_daemon_400/mcore2hf-config.vkfwHT --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 524288 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_400-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 400 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1012 15:04:29.820000 262631 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1012 15:04:29.820000 262631 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 400 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_daemon_400/mcore2hf-config.vkfwHT + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 524288 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_400-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_daemon_400/mcore2hf-config.vkfwHT + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.135 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_vkfwHT/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_vkfwHT/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_vkfwHT.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400 at iteration 400 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view400 [ t 1/1, p 1/1 ] at iteration 400 +[convert] synchronizer created in 392.55s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07763671875 mean=-0.000514984130859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10400390625 mean=0.00179290771484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07958984375 mean=-0.000152587890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10791015625 mean=0.00011777877807617188 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0859375 mean=-0.0002498626708984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.08056640625 mean=0.00080108642578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08935546875 mean=-0.00022029876708984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07958984375 mean=0.00145721435546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09765625 mean=-0.00023651123046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0751953125 mean=-0.000957489013671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0927734375 mean=-7.390975952148438e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0634765625 mean=-0.00014591217041015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0947265625 mean=-7.2479248046875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0634765625 mean=5.5789947509765625e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08935546875 mean=4.315376281738281e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05712890625 mean=-0.0003757476806640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08544921875 mean=4.2438507080078125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.054931640625 mean=-0.00028228759765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08349609375 mean=7.772445678710938e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06689453125 mean=-0.000476837158203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0771484375 mean=0.000225067138671875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0546875 mean=-0.000522613525390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07275390625 mean=0.00014591217041015625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043701171875 mean=-0.00113677978515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07666015625 mean=0.00017070770263671875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04736328125 mean=-0.000873565673828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07470703125 mean=0.000244140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.050048828125 mean=-0.000881195068359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07275390625 mean=0.00031280517578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.048583984375 mean=-0.00078582763671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0703125 mean=0.000316619873046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0478515625 mean=-0.001373291015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06640625 mean=0.0002841949462890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04833984375 mean=-0.002166748046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0693359375 mean=0.0006103515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.050048828125 mean=-0.00157928466796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06787109375 mean=0.0003662109375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.051025390625 mean=-0.00142669677734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060791015625 mean=0.000408172607421875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04248046875 mean=-0.002410888671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05615234375 mean=0.000324249267578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04443359375 mean=-0.0030364990234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05419921875 mean=0.0004596710205078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0380859375 mean=-0.002288818359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052978515625 mean=0.0004291534423828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0361328125 mean=-0.00262451171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.050048828125 mean=0.0003910064697265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03466796875 mean=-0.00182342529296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049560546875 mean=0.0003185272216796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0322265625 mean=-0.00174713134765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052001953125 mean=0.000446319580078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03076171875 mean=-0.001800537109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04833984375 mean=0.0004119873046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0277099609375 mean=-0.00099945068359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048828125 mean=0.00054931640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0244140625 mean=-0.00102996826171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052001953125 mean=0.000850677490234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02587890625 mean=-0.000949859619140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049560546875 mean=0.000530242919921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0267333984375 mean=-0.000568389892578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.046630859375 mean=0.0001373291015625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0257568359375 mean=-0.00016021728515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05029296875 mean=9.679794311523438e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.028076171875 mean=-3.337860107421875e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052490234375 mean=0.00017547607421875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0272216796875 mean=3.814697265625e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.057861328125 mean=6.29425048828125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.032470703125 mean=0.0003185272216796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06005859375 mean=6.079673767089844e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043212890625 mean=0.0009918212890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.059814453125 mean=0.00011205673217773438 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043212890625 mean=0.00140380859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.058349609375 mean=0.00012111663818359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03759765625 mean=0.0013885498046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.056884765625 mean=8.344650268554688e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.036865234375 mean=0.00113677978515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05322265625 mean=0.0001277923583984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.026123046875 mean=0.0002384185791015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=0.00012493133544921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0244140625 mean=0.00018787384033203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.044677734375 mean=0.000354766845703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0211181640625 mean=-0.00048065185546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038818359375 mean=0.00020122528076171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0205078125 mean=-0.000335693359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0361328125 mean=0.0004291534423828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0218505859375 mean=-0.000400543212890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03662109375 mean=0.00011682510375976562 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02392578125 mean=-0.000919342041015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036865234375 mean=0.000335693359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0211181640625 mean=-0.000598907470703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037109375 mean=0.00018978118896484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.023681640625 mean=-0.000820159912109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.034423828125 mean=-7.867813110351562e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.019775390625 mean=-0.0004787445068359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0341796875 mean=-0.00010538101196289062 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02197265625 mean=-0.00112152099609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0380859375 mean=-8.630752563476562e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.023681640625 mean=-0.0008697509765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038818359375 mean=4.291534423828125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0238037109375 mean=-0.00109100341796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04052734375 mean=0.00020885467529296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0302734375 mean=-0.00130462646484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0380859375 mean=9.72747802734375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.052001953125 mean=-0.0023651123046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0390625 mean=7.200241088867188e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0400390625 mean=-0.00201416015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.043701171875 mean=-0.00014209747314453125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043212890625 mean=0.0027618408203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.94s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_400-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 75.40s +Conversion finished in 470.357045173645 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_400-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_400-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_daemon_400/mcore2hf-config.vkfwHT ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter480.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter480.log new file mode 100644 index 0000000000000000000000000000000000000000..6c88346330b8ec6c1d7ff9da0863a7a7d1616259 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter480.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_daemon_480 ++ mkdir -p /tmp/mcore2hf_daemon_480 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 480 ']' ++ MANUAL_ITER=480 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480 ++ CHECKPOINT_BASENAME=load_view480 ++ [[ load_view480 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_daemon_480/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_daemon_480/mcore2hf-config.kwt5fd ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_daemon_480/mcore2hf-config.kwt5fd:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_daemon_480/mcore2hf-config.kwt5fd:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480 +W1012 20:07:02.019000 688367 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1012 20:07:02.019000 688367 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=524288 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=524288 +++ MAX_POSITION_EMBEDDINGS=524288 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480 +W1012 20:07:17.080000 688749 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1012 20:07:17.080000 688749 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 524288 ']' ++ '[' 524288 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_daemon_480/mcore2hf-config.kwt5fd +W1012 20:07:30.004000 689025 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1012 20:07:30.004000 689025 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_daemon_480/mcore2hf-config.kwt5fd/config.json' +✅config.json: /tmp/mcore2hf_daemon_480/mcore2hf-config.kwt5fd/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_daemon_480/mcore2hf-config.kwt5fd/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 524288, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_daemon_480/mcore2hf-config.kwt5fd ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480 480 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480 ++ local ITER_NUM=480 ++ '[' -z 480 ']' +++ printf %07d 480 ++ local ITER_BASENAME=iter_0000480 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480/iter_0000480 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480/iter_0000480 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480/latest_checkpointed_iteration.txt ++ LATEST_VALUE=480 ++ '[' 480 = 480 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=480 ++ echo -e '\033[32mLatest checkpoint iteration: 480\033[0m\n' +Latest checkpoint iteration: 480 + ++ I=480 ++ [[ 480 =~ ^[0-9]+$ ]] +++ printf %07d 480 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480/iter_0000480 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480/iter_0000480/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480/iter_480-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_480-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_480-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_480-hf true false bf16 /tmp/mcore2hf_daemon_480/mcore2hf-config.kwt5fd 480 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_480-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 35829 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_daemon_480/mcore2hf-config.kwt5fd --hf-dir /tmp/mcore2hf_daemon_480/mcore2hf-config.kwt5fd --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 524288 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_480-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 480 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1012 20:07:50.554000 689738 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1012 20:07:50.554000 689738 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 480 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_daemon_480/mcore2hf-config.kwt5fd + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 524288 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_480-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_daemon_480/mcore2hf-config.kwt5fd + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.137 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_kwt5fd/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_kwt5fd/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_kwt5fd.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480 at iteration 480 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view480 [ t 1/1, p 1/1 ] at iteration 480 +[convert] synchronizer created in 393.05s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07763671875 mean=-0.000514984130859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10400390625 mean=0.0015869140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07958984375 mean=-0.000141143798828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.107421875 mean=-1.1742115020751953e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0859375 mean=-0.00024318695068359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.08056640625 mean=0.000789642333984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08935546875 mean=-0.00022125244140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07958984375 mean=0.001373291015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09716796875 mean=-0.00022983551025390625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0751953125 mean=-0.00093841552734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09228515625 mean=-7.724761962890625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0634765625 mean=-0.0001392364501953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0947265625 mean=-7.200241088867188e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0634765625 mean=0.0002593994140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0888671875 mean=4.124641418457031e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.056884765625 mean=-0.000446319580078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0849609375 mean=4.506111145019531e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05517578125 mean=-0.0003223419189453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0830078125 mean=7.343292236328125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06640625 mean=-0.000354766845703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0771484375 mean=0.00022220611572265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.054931640625 mean=-0.0006256103515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.072265625 mean=0.0001373291015625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.044189453125 mean=-0.00113677978515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07666015625 mean=0.00016880035400390625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04736328125 mean=-0.000823974609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07421875 mean=0.00024127960205078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.050048828125 mean=-0.000789642333984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07275390625 mean=0.00031280517578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.048583984375 mean=-0.000728607177734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0703125 mean=0.0003185272216796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0478515625 mean=-0.00135040283203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06640625 mean=0.000286102294921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04833984375 mean=-0.00213623046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0693359375 mean=0.00060272216796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05029296875 mean=-0.001556396484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06787109375 mean=0.0003643035888671875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.051025390625 mean=-0.0014190673828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060791015625 mean=0.000396728515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042724609375 mean=-0.002471923828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.055908203125 mean=0.0003108978271484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.044677734375 mean=-0.0030364990234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.053955078125 mean=0.00045013427734375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.038330078125 mean=-0.002227783203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052734375 mean=0.0004215240478515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0361328125 mean=-0.0025634765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=0.00038909912109375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03466796875 mean=-0.00179290771484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04931640625 mean=0.0003185272216796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.032470703125 mean=-0.0016632080078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052001953125 mean=0.0004444122314453125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.031005859375 mean=-0.00183868408203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048095703125 mean=0.0004062652587890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0279541015625 mean=-0.000972747802734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048828125 mean=0.000545501708984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.024658203125 mean=-0.000972747802734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.000843048095703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0260009765625 mean=-0.00101470947265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04931640625 mean=0.000530242919921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.026611328125 mean=-0.000667572021484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.046630859375 mean=0.00013446807861328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.025634765625 mean=-0.000240325927734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05029296875 mean=9.679794311523438e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.028076171875 mean=-9.119510650634766e-06 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05224609375 mean=0.0001697540283203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0272216796875 mean=6.723403930664062e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.057861328125 mean=6.437301635742188e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.032470703125 mean=0.000217437744140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.059814453125 mean=5.936622619628906e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04345703125 mean=0.000942230224609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0595703125 mean=0.00011301040649414062 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043212890625 mean=0.001373291015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.058349609375 mean=0.00011920928955078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03759765625 mean=0.00150299072265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.056640625 mean=8.630752563476562e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.036865234375 mean=0.00122833251953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052978515625 mean=0.0001316070556640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0260009765625 mean=0.0002880096435546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=0.00012302398681640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0244140625 mean=7.581710815429688e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04443359375 mean=0.000354766845703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0211181640625 mean=-0.000507354736328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038818359375 mean=0.00020313262939453125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0205078125 mean=-0.000347137451171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0361328125 mean=0.000431060791015625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0218505859375 mean=-0.000385284423828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036376953125 mean=0.0001163482666015625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0240478515625 mean=-0.000934600830078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03662109375 mean=0.0003337860107421875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.021240234375 mean=-0.00045013427734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036865234375 mean=0.00018978118896484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02392578125 mean=-0.000843048095703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0341796875 mean=-7.343292236328125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.019775390625 mean=-0.000476837158203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0341796875 mean=-0.00010156631469726562 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02197265625 mean=-0.0010223388671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=-7.152557373046875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0238037109375 mean=-0.000797271728515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0390625 mean=5.793571472167969e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0238037109375 mean=-0.00110626220703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04052734375 mean=0.0002117156982421875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0303955078125 mean=-0.00127410888671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037841796875 mean=9.298324584960938e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.052734375 mean=-0.002410888671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0390625 mean=8.058547973632812e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04052734375 mean=-0.002044677734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.043701171875 mean=-0.000141143798828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04345703125 mean=0.002716064453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.95s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_480-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 71.36s +Conversion finished in 466.70248794555664 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_480-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_480-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_daemon_480/mcore2hf-config.kwt5fd ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter560.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter560.log new file mode 100644 index 0000000000000000000000000000000000000000..e7d89e3c70e23a45c0a190a10dbef9e8148a44cb --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter560.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_daemon_560 ++ mkdir -p /tmp/mcore2hf_daemon_560 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 560 ']' ++ MANUAL_ITER=560 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560 ++ CHECKPOINT_BASENAME=load_view560 ++ [[ load_view560 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_daemon_560/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_daemon_560/mcore2hf-config.ilhsUl ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_daemon_560/mcore2hf-config.ilhsUl:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_daemon_560/mcore2hf-config.ilhsUl:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560 +W1013 01:10:21.235000 16106 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1013 01:10:21.235000 16106 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=524288 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=524288 +++ MAX_POSITION_EMBEDDINGS=524288 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560 +W1013 01:10:36.280000 17066 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1013 01:10:36.280000 17066 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 524288 ']' ++ '[' 524288 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_daemon_560/mcore2hf-config.ilhsUl +W1013 01:10:49.538000 17334 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1013 01:10:49.538000 17334 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_daemon_560/mcore2hf-config.ilhsUl/config.json' +✅config.json: /tmp/mcore2hf_daemon_560/mcore2hf-config.ilhsUl/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_daemon_560/mcore2hf-config.ilhsUl/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 524288, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_daemon_560/mcore2hf-config.ilhsUl ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560 560 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560 ++ local ITER_NUM=560 ++ '[' -z 560 ']' +++ printf %07d 560 ++ local ITER_BASENAME=iter_0000560 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560/iter_0000560 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560/iter_0000560 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560/latest_checkpointed_iteration.txt ++ LATEST_VALUE=560 ++ '[' 560 = 560 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=560 ++ echo -e '\033[32mLatest checkpoint iteration: 560\033[0m\n' +Latest checkpoint iteration: 560 + ++ I=560 ++ [[ 560 =~ ^[0-9]+$ ]] +++ printf %07d 560 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560/iter_0000560 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560/iter_0000560/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560/iter_560-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_560-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_560-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_560-hf true false bf16 /tmp/mcore2hf_daemon_560/mcore2hf-config.ilhsUl 560 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_560-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 38173 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_daemon_560/mcore2hf-config.ilhsUl --hf-dir /tmp/mcore2hf_daemon_560/mcore2hf-config.ilhsUl --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 524288 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_560-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 560 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1013 01:11:11.830000 17854 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1013 01:11:11.830000 17854 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 560 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_daemon_560/mcore2hf-config.ilhsUl + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 524288 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_560-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_daemon_560/mcore2hf-config.ilhsUl + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.136 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_ilhsUl/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_ilhsUl/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_ilhsUl.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560 at iteration 560 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view560 [ t 1/1, p 1/1 ] at iteration 560 +[convert] synchronizer created in 385.33s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0771484375 mean=-0.00051116943359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.103515625 mean=0.0015869140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07958984375 mean=-0.000141143798828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.107421875 mean=-4.649162292480469e-06 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0859375 mean=-0.0002460479736328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.08056640625 mean=0.00084686279296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08935546875 mean=-0.0002231597900390625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07958984375 mean=0.0013580322265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09716796875 mean=-0.0002288818359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07470703125 mean=-0.0007476806640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09228515625 mean=-7.581710815429688e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06298828125 mean=-0.0001201629638671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0947265625 mean=-6.961822509765625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0634765625 mean=0.0001773834228515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0888671875 mean=4.124641418457031e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.056884765625 mean=-0.0003108978271484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0849609375 mean=4.6253204345703125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05517578125 mean=-0.0002899169921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0830078125 mean=7.2479248046875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06640625 mean=-0.000354766845703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07666015625 mean=0.00022220611572265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0546875 mean=-0.000614166259765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.072265625 mean=0.0001354217529296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0439453125 mean=-0.000919342041015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07666015625 mean=0.000171661376953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.047119140625 mean=-0.000896453857421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07421875 mean=0.00023746490478515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.050048828125 mean=-0.000751495361328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07275390625 mean=0.0003108978271484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.048828125 mean=-0.00069427490234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0703125 mean=0.0003185272216796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.047607421875 mean=-0.00140380859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06640625 mean=0.0002841949462890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04833984375 mean=-0.0022125244140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0693359375 mean=0.00060272216796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05029296875 mean=-0.0016326904296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06787109375 mean=0.0003643035888671875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.051025390625 mean=-0.00131988525390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060791015625 mean=0.0003986358642578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04248046875 mean=-0.0024566650390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.055908203125 mean=0.0003147125244140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.044677734375 mean=-0.0030517578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.053955078125 mean=0.00045013427734375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0380859375 mean=-0.0022735595703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052734375 mean=0.000423431396484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.035888671875 mean=-0.0025634765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=0.0003871917724609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.034423828125 mean=-0.00180816650390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04931640625 mean=0.0003185272216796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0322265625 mean=-0.001708984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.0004425048828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0308837890625 mean=-0.0018157958984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048095703125 mean=0.000408172607421875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0277099609375 mean=-0.00102996826171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048583984375 mean=0.000545501708984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.024658203125 mean=-0.000934600830078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.000843048095703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02587890625 mean=-0.0010528564453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=0.00054168701171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.026611328125 mean=-0.000682830810546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.046630859375 mean=0.00013637542724609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0257568359375 mean=-0.0003204345703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.050048828125 mean=9.489059448242188e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.028076171875 mean=-7.581710815429688e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05224609375 mean=0.00017070770263671875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02734375 mean=9.298324584960938e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0576171875 mean=6.580352783203125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03271484375 mean=0.0002841949462890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0595703125 mean=6.079673767089844e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04345703125 mean=0.0009918212890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.059326171875 mean=0.00011444091796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043212890625 mean=0.00136566162109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05810546875 mean=0.00011920928955078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03759765625 mean=0.00147247314453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.056640625 mean=8.535385131835938e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.036865234375 mean=0.00119781494140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052978515625 mean=0.00013065338134765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0260009765625 mean=0.00045013427734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049560546875 mean=0.00012111663818359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0244140625 mean=0.000156402587890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04443359375 mean=0.0003490447998046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.021240234375 mean=-0.000423431396484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03857421875 mean=0.00020122528076171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0203857421875 mean=-0.0003147125244140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.035888671875 mean=0.00043487548828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0218505859375 mean=-0.0004177093505859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036376953125 mean=0.00011491775512695312 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0240478515625 mean=-0.00084686279296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036376953125 mean=0.00032806396484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0211181640625 mean=-0.000576019287109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036865234375 mean=0.00018405914306640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02392578125 mean=-0.000911712646484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0341796875 mean=-7.200241088867188e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.019775390625 mean=-0.00057220458984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0341796875 mean=-0.00011157989501953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0218505859375 mean=-0.001068115234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=-7.104873657226562e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02392578125 mean=-0.000762939453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0390625 mean=6.961822509765625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0238037109375 mean=-0.0011749267578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.040283203125 mean=0.00020694732666015625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.030517578125 mean=-0.00138092041015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037841796875 mean=9.918212890625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.052978515625 mean=-0.0024566650390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038818359375 mean=7.963180541992188e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.040771484375 mean=-0.002197265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04345703125 mean=-0.00014019012451171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043701171875 mean=0.00274658203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.95s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_560-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 73.74s +Conversion finished in 461.3664696216583 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_560-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_560-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_daemon_560/mcore2hf-config.ilhsUl ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter640.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter640.log new file mode 100644 index 0000000000000000000000000000000000000000..d6ddd1462cc82eea912f2d5dc5353bc6367a6d7e --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter640.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_daemon_640 ++ mkdir -p /tmp/mcore2hf_daemon_640 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 640 ']' ++ MANUAL_ITER=640 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640 ++ CHECKPOINT_BASENAME=load_view640 ++ [[ load_view640 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_daemon_640/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_daemon_640/mcore2hf-config.Ugb1cn ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_daemon_640/mcore2hf-config.Ugb1cn:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_daemon_640/mcore2hf-config.Ugb1cn:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640 +W1013 06:11:56.228000 329467 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1013 06:11:56.228000 329467 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=524288 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=524288 +++ MAX_POSITION_EMBEDDINGS=524288 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640 +W1013 06:12:11.500000 329852 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1013 06:12:11.500000 329852 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 524288 ']' ++ '[' 524288 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_daemon_640/mcore2hf-config.Ugb1cn +W1013 06:12:24.328000 330241 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1013 06:12:24.328000 330241 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_daemon_640/mcore2hf-config.Ugb1cn/config.json' +✅config.json: /tmp/mcore2hf_daemon_640/mcore2hf-config.Ugb1cn/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_daemon_640/mcore2hf-config.Ugb1cn/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 524288, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_daemon_640/mcore2hf-config.Ugb1cn ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640 640 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640 ++ local ITER_NUM=640 ++ '[' -z 640 ']' +++ printf %07d 640 ++ local ITER_BASENAME=iter_0000640 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640/iter_0000640 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640/iter_0000640 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640/latest_checkpointed_iteration.txt ++ LATEST_VALUE=640 ++ '[' 640 = 640 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=640 ++ echo -e '\033[32mLatest checkpoint iteration: 640\033[0m\n' +Latest checkpoint iteration: 640 + ++ I=640 ++ [[ 640 =~ ^[0-9]+$ ]] +++ printf %07d 640 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640/iter_0000640 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640/iter_0000640/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640/iter_640-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_640-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_640-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_640-hf true false bf16 /tmp/mcore2hf_daemon_640/mcore2hf-config.Ugb1cn 640 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_640-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 39289 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_daemon_640/mcore2hf-config.Ugb1cn --hf-dir /tmp/mcore2hf_daemon_640/mcore2hf-config.Ugb1cn --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 524288 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_640-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 640 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1013 06:12:44.538000 330790 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1013 06:12:44.538000 330790 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 640 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_daemon_640/mcore2hf-config.Ugb1cn + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 524288 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_640-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_daemon_640/mcore2hf-config.Ugb1cn + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.130 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_Ugb1cn/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_Ugb1cn/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_Ugb1cn.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640 at iteration 640 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view640 [ t 1/1, p 1/1 ] at iteration 640 +[convert] synchronizer created in 390.62s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0771484375 mean=-0.000514984130859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.103515625 mean=0.001708984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07958984375 mean=-0.00011491775512695312 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.107421875 mean=0.000186920166015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08544921875 mean=-0.000217437744140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.08056640625 mean=0.000865936279296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0888671875 mean=-0.00022220611572265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07958984375 mean=0.00145721435546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09716796875 mean=-0.00022220611572265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07470703125 mean=-0.00069427490234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09228515625 mean=-7.724761962890625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0634765625 mean=-0.00015735626220703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09423828125 mean=-6.866455078125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06396484375 mean=0.00014781951904296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0888671875 mean=4.100799560546875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05712890625 mean=-0.000270843505859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0849609375 mean=5.6743621826171875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05517578125 mean=-0.000324249267578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0830078125 mean=5.1975250244140625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06640625 mean=-0.000255584716796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07666015625 mean=0.00020885467529296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.054931640625 mean=-0.000606536865234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.072265625 mean=0.00011157989501953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0439453125 mean=-0.00090789794921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07666015625 mean=0.00015354156494140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.047119140625 mean=-0.0008697509765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07421875 mean=0.0002269744873046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.050048828125 mean=-0.00079345703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07275390625 mean=0.00029754638671875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.048583984375 mean=-0.0007781982421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06982421875 mean=0.00031280517578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.047607421875 mean=-0.0014495849609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06591796875 mean=0.000270843505859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.048095703125 mean=-0.002227783203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06884765625 mean=0.00057220458984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.050048828125 mean=-0.001678466796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06787109375 mean=0.0003566741943359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.051025390625 mean=-0.00131988525390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060791015625 mean=0.0003833770751953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04248046875 mean=-0.0025787353515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.055908203125 mean=0.0002956390380859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.044677734375 mean=-0.0030059814453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.053955078125 mean=0.0004520416259765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.038330078125 mean=-0.0022125244140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052734375 mean=0.00041961669921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.035888671875 mean=-0.002593994140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=0.000377655029296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.034423828125 mean=-0.0018310546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04931640625 mean=0.0003108978271484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0322265625 mean=-0.0015869140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.0004367828369140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.031005859375 mean=-0.001708984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0478515625 mean=0.0004024505615234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0277099609375 mean=-0.00103759765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048583984375 mean=0.0005340576171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.024658203125 mean=-0.00089263916015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.000835418701171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0260009765625 mean=-0.00113677978515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=0.0005340576171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.026611328125 mean=-0.000579833984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04638671875 mean=0.00012874603271484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0257568359375 mean=-0.0003261566162109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.050048828125 mean=8.726119995117188e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.028076171875 mean=-4.9114227294921875e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05224609375 mean=0.0001621246337890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02734375 mean=3.314018249511719e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0576171875 mean=5.14984130859375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03271484375 mean=0.0003185272216796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0595703125 mean=5.53131103515625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04345703125 mean=0.000911712646484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.059326171875 mean=0.00010585784912109375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04296875 mean=0.00142669677734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05810546875 mean=0.0001125335693359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03759765625 mean=0.00136566162109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.056396484375 mean=7.772445678710938e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037109375 mean=0.00116729736328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052978515625 mean=0.0001220703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0260009765625 mean=0.0003223419189453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049560546875 mean=0.00011157989501953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0245361328125 mean=9.489059448242188e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04443359375 mean=0.0003337860107421875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.021240234375 mean=-0.000431060791015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03857421875 mean=0.0001888275146484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0203857421875 mean=-0.000339508056640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.035888671875 mean=0.0004253387451171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02197265625 mean=-0.0004253387451171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036376953125 mean=0.00010156631469726562 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0240478515625 mean=-0.0008697509765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036376953125 mean=0.000316619873046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.021240234375 mean=-0.000537872314453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036865234375 mean=0.00018215179443359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02392578125 mean=-0.000942230224609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0341796875 mean=-8.296966552734375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0198974609375 mean=-0.000522613525390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0341796875 mean=-0.00012159347534179688 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02197265625 mean=-0.001068115234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=-9.393692016601562e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.024169921875 mean=-0.000736236572265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03955078125 mean=3.0279159545898438e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02392578125 mean=-0.00118255615234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.040283203125 mean=0.0001964569091796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03076171875 mean=-0.001251220703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037841796875 mean=9.679794311523438e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05322265625 mean=-0.00250244140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038818359375 mean=8.58306884765625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.041015625 mean=-0.0020904541015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04345703125 mean=-0.000133514404296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.044189453125 mean=0.002777099609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.94s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_640-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 81.84s +Conversion finished in 474.8521809577942 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_640-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_640-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_daemon_640/mcore2hf-config.Ugb1cn ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter720.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter720.log new file mode 100644 index 0000000000000000000000000000000000000000..137b12356bb10fbbf009787d61dd03405e6d6b68 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter720.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_daemon_720 ++ mkdir -p /tmp/mcore2hf_daemon_720 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 720 ']' ++ MANUAL_ITER=720 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720 ++ CHECKPOINT_BASENAME=load_view720 ++ [[ load_view720 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_daemon_720/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_daemon_720/mcore2hf-config.jsiHLl ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_daemon_720/mcore2hf-config.jsiHLl:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_daemon_720/mcore2hf-config.jsiHLl:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720 +W1013 11:15:09.428000 681601 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1013 11:15:09.428000 681601 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=524288 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=524288 +++ MAX_POSITION_EMBEDDINGS=524288 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720 +W1013 11:15:22.318000 681826 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1013 11:15:22.318000 681826 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 524288 ']' ++ '[' 524288 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_daemon_720/mcore2hf-config.jsiHLl +W1013 11:15:35.365000 682200 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1013 11:15:35.365000 682200 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_daemon_720/mcore2hf-config.jsiHLl/config.json' +✅config.json: /tmp/mcore2hf_daemon_720/mcore2hf-config.jsiHLl/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_daemon_720/mcore2hf-config.jsiHLl/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 524288, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_daemon_720/mcore2hf-config.jsiHLl ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720 720 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720 ++ local ITER_NUM=720 ++ '[' -z 720 ']' +++ printf %07d 720 ++ local ITER_BASENAME=iter_0000720 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720/iter_0000720 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720/iter_0000720 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720/latest_checkpointed_iteration.txt ++ LATEST_VALUE=720 ++ '[' 720 = 720 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=720 ++ echo -e '\033[32mLatest checkpoint iteration: 720\033[0m\n' +Latest checkpoint iteration: 720 + ++ I=720 ++ [[ 720 =~ ^[0-9]+$ ]] +++ printf %07d 720 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720/iter_0000720 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720/iter_0000720/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720/iter_720-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_720-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_720-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_720-hf true false bf16 /tmp/mcore2hf_daemon_720/mcore2hf-config.jsiHLl 720 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_720-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 33075 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_daemon_720/mcore2hf-config.jsiHLl --hf-dir /tmp/mcore2hf_daemon_720/mcore2hf-config.jsiHLl --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 524288 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_720-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 720 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1013 11:15:56.021000 682604 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1013 11:15:56.021000 682604 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 720 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_daemon_720/mcore2hf-config.jsiHLl + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 524288 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_720-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_daemon_720/mcore2hf-config.jsiHLl + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.132 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_jsiHLl/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_jsiHLl/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_jsiHLl.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720 at iteration 720 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view720 [ t 1/1, p 1/1 ] at iteration 720 +[convert] synchronizer created in 385.92s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0771484375 mean=-0.00051116943359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.103515625 mean=0.0016632080078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07958984375 mean=-0.0001201629638671875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10693359375 mean=0.00013065338134765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08544921875 mean=-0.00021266937255859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.08056640625 mean=0.00095367431640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0888671875 mean=-0.0002269744873046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0791015625 mean=0.001495361328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0966796875 mean=-0.000217437744140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07470703125 mean=-0.000743865966796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.091796875 mean=-7.915496826171875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0634765625 mean=-0.0001201629638671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09423828125 mean=-7.104873657226562e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0634765625 mean=5.888938903808594e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08837890625 mean=3.838539123535156e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05712890625 mean=-0.000385284423828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08447265625 mean=5.793571472167969e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.054931640625 mean=-0.00042724609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0830078125 mean=4.029273986816406e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06640625 mean=-0.0003452301025390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07666015625 mean=0.00020599365234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05517578125 mean=-0.000579833984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.072265625 mean=0.00011396408081054688 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.044189453125 mean=-0.000843048095703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.000156402587890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04736328125 mean=-0.000885009765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07421875 mean=0.0002288818359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.050048828125 mean=-0.00077056884765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07275390625 mean=0.000301361083984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.048583984375 mean=-0.000583648681640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06982421875 mean=0.00031280517578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.047607421875 mean=-0.00146484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06591796875 mean=0.000270843505859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0478515625 mean=-0.0022125244140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06884765625 mean=0.000568389892578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.050048828125 mean=-0.0015869140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06787109375 mean=0.0003566741943359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.051025390625 mean=-0.00128173828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060546875 mean=0.000377655029296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042236328125 mean=-0.00244140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.055908203125 mean=0.0002918243408203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.044677734375 mean=-0.003021240234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.053955078125 mean=0.0004520416259765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.038330078125 mean=-0.0023040771484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052490234375 mean=0.00041961669921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0361328125 mean=-0.0025482177734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=0.0003814697265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.034423828125 mean=-0.00173187255859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=0.000308990478515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.031982421875 mean=-0.0014495849609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.051513671875 mean=0.0004329681396484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0308837890625 mean=-0.0017242431640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0478515625 mean=0.0003986358642578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.027587890625 mean=-0.00101470947265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048583984375 mean=0.000530242919921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0245361328125 mean=-0.00090789794921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.00081634521484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0260009765625 mean=-0.0010833740234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=0.0005340576171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.026611328125 mean=-0.000576019287109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04638671875 mean=0.00013065338134765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0257568359375 mean=-0.00030517578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.050048828125 mean=8.678436279296875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.028076171875 mean=-1.4066696166992188e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052001953125 mean=0.00016117095947265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0272216796875 mean=0.00010061264038085938 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0576171875 mean=4.744529724121094e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.032470703125 mean=0.0003376007080078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0595703125 mean=4.9591064453125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04345703125 mean=0.00081634521484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.059326171875 mean=0.00010538101196289062 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043212890625 mean=0.00128936767578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.057861328125 mean=0.00010919570922851562 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037353515625 mean=0.00152587890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.056396484375 mean=7.486343383789062e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037109375 mean=0.0012054443359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052734375 mean=0.000118255615234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0260009765625 mean=0.0004253387451171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049560546875 mean=0.00010824203491210938 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0245361328125 mean=0.00018978118896484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.044189453125 mean=0.0003337860107421875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.021240234375 mean=-0.00052642822265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03857421875 mean=0.00018215179443359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0205078125 mean=-0.0002288818359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.035888671875 mean=0.000431060791015625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0220947265625 mean=-0.000415802001953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0361328125 mean=0.000102996826171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0242919921875 mean=-0.000904083251953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036376953125 mean=0.00031280517578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0211181640625 mean=-0.0006103515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036865234375 mean=0.000186920166015625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0240478515625 mean=-0.000911712646484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.033935546875 mean=-7.724761962890625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0201416015625 mean=-0.000499725341796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0341796875 mean=-0.00011777877807617188 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0220947265625 mean=-0.0010986328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=-9.632110595703125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0242919921875 mean=-0.00072479248046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03955078125 mean=2.574920654296875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0240478515625 mean=-0.00113677978515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.040283203125 mean=0.00019550323486328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0308837890625 mean=-0.0013427734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03759765625 mean=9.250640869140625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05322265625 mean=-0.002410888671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038818359375 mean=8.96453857421875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.041259765625 mean=-0.0020294189453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04345703125 mean=-0.00013065338134765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0439453125 mean=0.0028228759765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.95s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_720-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 81.69s +Conversion finished in 469.97148990631104 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_720-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_720-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_daemon_720/mcore2hf-config.jsiHLl ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter80.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter80.log new file mode 100644 index 0000000000000000000000000000000000000000..1a54f3d835c9db76d4ea4da4b3e984b82dd799ce --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter80.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_daemon_80 ++ mkdir -p /tmp/mcore2hf_daemon_80 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 80 ']' ++ MANUAL_ITER=80 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80 ++ CHECKPOINT_BASENAME=load_view80 ++ [[ load_view80 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_daemon_80/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_daemon_80/mcore2hf-config.xqCYmb ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_daemon_80/mcore2hf-config.xqCYmb:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_daemon_80/mcore2hf-config.xqCYmb:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80 +W1011 18:52:52.544000 944436 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 18:52:52.544000 944436 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=524288 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=524288 +++ MAX_POSITION_EMBEDDINGS=524288 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80 +W1011 18:53:06.114000 944779 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 18:53:06.114000 944779 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 524288 ']' ++ '[' 524288 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_daemon_80/mcore2hf-config.xqCYmb +W1011 18:53:19.206000 945014 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 18:53:19.206000 945014 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_daemon_80/mcore2hf-config.xqCYmb/config.json' +✅config.json: /tmp/mcore2hf_daemon_80/mcore2hf-config.xqCYmb/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_daemon_80/mcore2hf-config.xqCYmb/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 524288, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_daemon_80/mcore2hf-config.xqCYmb ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80 80 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80 ++ local ITER_NUM=80 ++ '[' -z 80 ']' +++ printf %07d 80 ++ local ITER_BASENAME=iter_0000080 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80/iter_0000080 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80/iter_0000080 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80/latest_checkpointed_iteration.txt ++ LATEST_VALUE=80 ++ '[' 80 = 80 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=80 ++ echo -e '\033[32mLatest checkpoint iteration: 80\033[0m\n' +Latest checkpoint iteration: 80 + ++ I=80 ++ [[ 80 =~ ^[0-9]+$ ]] +++ printf %07d 80 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80/iter_0000080 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80/iter_0000080/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80/iter_80-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_80-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_80-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_80-hf true false bf16 /tmp/mcore2hf_daemon_80/mcore2hf-config.xqCYmb 80 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_80-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 46843 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_daemon_80/mcore2hf-config.xqCYmb --hf-dir /tmp/mcore2hf_daemon_80/mcore2hf-config.xqCYmb --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 524288 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_80-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 80 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1011 18:53:39.097000 945451 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 18:53:39.097000 945451 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 80 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_daemon_80/mcore2hf-config.xqCYmb + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 524288 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_80-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_daemon_80/mcore2hf-config.xqCYmb + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.129 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_xqCYmb/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_xqCYmb/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_xqCYmb.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80 at iteration 80 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view80 [ t 1/1, p 1/1 ] at iteration 80 +[convert] synchronizer created in 385.69s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.078125 mean=-0.000499725341796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10498046875 mean=0.0018463134765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.080078125 mean=-0.00015926361083984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.1083984375 mean=-4.029273986816406e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0869140625 mean=-0.00026702880859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.080078125 mean=0.000782012939453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09033203125 mean=-0.00020694732666015625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07958984375 mean=0.001312255859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09814453125 mean=-0.000232696533203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07421875 mean=-0.000850677490234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09326171875 mean=-7.295608520507812e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06298828125 mean=-0.0002193450927734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.095703125 mean=-7.343292236328125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06298828125 mean=1.3649463653564453e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08984375 mean=3.0040740966796875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.055908203125 mean=-0.0002002716064453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08642578125 mean=2.586841583251953e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.053955078125 mean=-0.000518798828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.083984375 mean=7.200241088867188e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0654296875 mean=-0.0003833770751953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.078125 mean=0.00022411346435546875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05322265625 mean=-0.000583648681640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07373046875 mean=0.00015163421630859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04296875 mean=-0.000949859619140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07763671875 mean=0.00016689300537109375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.046142578125 mean=-0.000690460205078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0751953125 mean=0.00023174285888671875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.048828125 mean=-0.0010528564453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07373046875 mean=0.0002956390380859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0478515625 mean=-0.00083160400390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0712890625 mean=0.0002994537353515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.046875 mean=-0.00135040283203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0673828125 mean=0.0002803802490234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04736328125 mean=-0.0022430419921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0703125 mean=0.000614166259765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04931640625 mean=-0.00164794921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06884765625 mean=0.000385284423828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.049560546875 mean=-0.0010528564453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0615234375 mean=0.00041961669921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.041259765625 mean=-0.002166748046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.056640625 mean=0.0003528594970703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0439453125 mean=-0.0030517578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.054931640625 mean=0.0004787445068359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037109375 mean=-0.0021514892578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0537109375 mean=0.0004405975341796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.035400390625 mean=-0.002471923828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05078125 mean=0.00040435791015625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.033447265625 mean=-0.001739501953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05029296875 mean=0.0003337860107421875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03125 mean=-0.00176239013671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052734375 mean=0.0004558563232421875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0294189453125 mean=-0.00173187255859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=0.0004062652587890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.027099609375 mean=-0.0009765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049560546875 mean=0.00054931640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0235595703125 mean=-0.00107574462890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052734375 mean=0.000850677490234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0247802734375 mean=-0.00089263916015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05029296875 mean=0.0005340576171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0257568359375 mean=-0.0005340576171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.047607421875 mean=0.00013446807861328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02490234375 mean=-0.00014019012451171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.051025390625 mean=9.632110595703125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02734375 mean=-7.152557373046875e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05322265625 mean=0.0001850128173828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0267333984375 mean=0.00011587142944335938 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05859375 mean=7.915496826171875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03173828125 mean=0.0003719329833984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060791015625 mean=5.3882598876953125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042724609375 mean=0.00103759765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060546875 mean=0.00011157989501953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042724609375 mean=0.00130462646484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05908203125 mean=0.00012302398681640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037109375 mean=0.0015869140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0576171875 mean=8.58306884765625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03662109375 mean=0.00127410888671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.053955078125 mean=0.00012969970703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0255126953125 mean=0.000286102294921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.050537109375 mean=0.00013637542724609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0240478515625 mean=0.00021076202392578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04541015625 mean=0.00037384033203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0205078125 mean=-0.0006256103515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03955078125 mean=0.00019931793212890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02001953125 mean=-0.000392913818359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03662109375 mean=0.000423431396484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02099609375 mean=-0.0003948211669921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037109375 mean=0.00012683868408203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.023193359375 mean=-0.000827789306640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037353515625 mean=0.0003566741943359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0205078125 mean=-0.000640869140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03759765625 mean=0.0002155303955078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02294921875 mean=-0.000720977783203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.034912109375 mean=-7.390975952148438e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0191650390625 mean=-0.00054168701171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03466796875 mean=-0.0001049041748046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0211181640625 mean=-0.00099945068359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0380859375 mean=-0.0001125335693359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0224609375 mean=-0.000774383544921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0380859375 mean=1.9431114196777344e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0228271484375 mean=-0.0011138916015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04052734375 mean=0.00021266937255859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0291748046875 mean=-0.0012969970703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038818359375 mean=7.963180541992188e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.050048828125 mean=-0.002288818359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.039794921875 mean=7.05718994140625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0380859375 mean=-0.0019683837890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04443359375 mean=-0.0001468658447265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.041748046875 mean=0.0024261474609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.99s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_80-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 77.04s +Conversion finished in 465.0616121292114 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_80-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_80-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_daemon_80/mcore2hf-config.xqCYmb ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter880.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter880.log new file mode 100644 index 0000000000000000000000000000000000000000..780f5b40beab5f660048eb59c2f2ab51c7abfe5a --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter880.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_daemon_880 ++ mkdir -p /tmp/mcore2hf_daemon_880 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 880 ']' ++ MANUAL_ITER=880 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880 ++ CHECKPOINT_BASENAME=load_view880 ++ [[ load_view880 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_daemon_880/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_daemon_880/mcore2hf-config.n9XkHZ ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_daemon_880/mcore2hf-config.n9XkHZ:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_daemon_880/mcore2hf-config.n9XkHZ:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880 +W1013 21:17:42.771000 408063 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1013 21:17:42.771000 408063 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=524288 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=524288 +++ MAX_POSITION_EMBEDDINGS=524288 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880 +W1013 21:18:04.985000 409263 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1013 21:18:04.985000 409263 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 524288 ']' ++ '[' 524288 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_daemon_880/mcore2hf-config.n9XkHZ +W1013 21:18:18.635000 409583 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1013 21:18:18.635000 409583 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_daemon_880/mcore2hf-config.n9XkHZ/config.json' +✅config.json: /tmp/mcore2hf_daemon_880/mcore2hf-config.n9XkHZ/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_daemon_880/mcore2hf-config.n9XkHZ/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 524288, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_daemon_880/mcore2hf-config.n9XkHZ ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880 880 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880 ++ local ITER_NUM=880 ++ '[' -z 880 ']' +++ printf %07d 880 ++ local ITER_BASENAME=iter_0000880 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880/iter_0000880 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880/iter_0000880 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880/latest_checkpointed_iteration.txt ++ LATEST_VALUE=880 ++ '[' 880 = 880 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=880 ++ echo -e '\033[32mLatest checkpoint iteration: 880\033[0m\n' +Latest checkpoint iteration: 880 + ++ I=880 ++ [[ 880 =~ ^[0-9]+$ ]] +++ printf %07d 880 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880/iter_0000880 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880/iter_0000880/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880/iter_880-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_880-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_880-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_880-hf true false bf16 /tmp/mcore2hf_daemon_880/mcore2hf-config.n9XkHZ 880 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_880-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 52273 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_daemon_880/mcore2hf-config.n9XkHZ --hf-dir /tmp/mcore2hf_daemon_880/mcore2hf-config.n9XkHZ --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 524288 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_880-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 880 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1013 21:18:39.202000 410063 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1013 21:18:39.202000 410063 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 880 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_daemon_880/mcore2hf-config.n9XkHZ + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 524288 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_880-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_daemon_880/mcore2hf-config.n9XkHZ + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.138 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_n9XkHZ/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_n9XkHZ/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_n9XkHZ.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880 at iteration 880 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view880 [ t 1/1, p 1/1 ] at iteration 880 +[convert] synchronizer created in 394.13s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0771484375 mean=-0.000499725341796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10302734375 mean=0.001556396484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0791015625 mean=-0.00011396408081054688 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.1064453125 mean=-0.00011682510375976562 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08544921875 mean=-0.00021457672119140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.08056640625 mean=0.000835418701171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08837890625 mean=-0.00022411346435546875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07861328125 mean=0.00154876708984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0966796875 mean=-0.000213623046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07470703125 mean=-0.000850677490234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.091796875 mean=-7.62939453125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06298828125 mean=-0.000186920166015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09423828125 mean=-7.390975952148438e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0634765625 mean=0.0001811981201171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08837890625 mean=4.076957702636719e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.056884765625 mean=-0.000339508056640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08447265625 mean=5.6743621826171875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05517578125 mean=-0.0005035400390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08251953125 mean=3.3855438232421875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06640625 mean=-0.0004253387451171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.0001983642578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.054931640625 mean=-0.000606536865234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07177734375 mean=0.00010824203491210938 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043701171875 mean=-0.00095367431640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.000152587890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04736328125 mean=-0.0010528564453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07373046875 mean=0.0002288818359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0498046875 mean=-0.0008697509765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07275390625 mean=0.0002994537353515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04833984375 mean=-0.0005035400390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06982421875 mean=0.00031280517578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.047119140625 mean=-0.0015411376953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06591796875 mean=0.00026702880859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0478515625 mean=-0.0020904541015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06884765625 mean=0.000560760498046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0498046875 mean=-0.00160980224609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06787109375 mean=0.0003452301025390625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05078125 mean=-0.00130462646484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060546875 mean=0.0003719329833984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042236328125 mean=-0.002349853515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0556640625 mean=0.0002956390380859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04443359375 mean=-0.003082275390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0537109375 mean=0.000446319580078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0380859375 mean=-0.00225830078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052490234375 mean=0.0004253387451171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.035888671875 mean=-0.00250244140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049560546875 mean=0.0003795623779296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.034423828125 mean=-0.00177001953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=0.00030517578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03173828125 mean=-0.00159454345703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.051513671875 mean=0.0004253387451171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03076171875 mean=-0.00177764892578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0478515625 mean=0.000392913818359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02734375 mean=-0.001007080078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04833984375 mean=0.00052642822265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0245361328125 mean=-0.000804901123046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.0008087158203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02587890625 mean=-0.00098419189453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048828125 mean=0.0005340576171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02685546875 mean=-0.000522613525390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04638671875 mean=0.00012874603271484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.025634765625 mean=-0.000415802001953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=8.630752563476562e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0281982421875 mean=-3.552436828613281e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052001953125 mean=0.00016021728515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.027099609375 mean=-3.790855407714844e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.057373046875 mean=4.863739013671875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.032470703125 mean=0.00040435791015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.059326171875 mean=4.7206878662109375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043212890625 mean=0.000926971435546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05908203125 mean=0.00010013580322265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04296875 mean=0.00118255615234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.057861328125 mean=0.00010728836059570312 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037353515625 mean=0.00145721435546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05615234375 mean=6.628036499023438e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037109375 mean=0.00115966796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052490234375 mean=0.00010728836059570312 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02587890625 mean=0.000293731689453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04931640625 mean=9.72747802734375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0242919921875 mean=0.000255584716796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.044189453125 mean=0.0003261566162109375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0211181640625 mean=-0.000522613525390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=0.000171661376953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0203857421875 mean=-0.0001888275146484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.035888671875 mean=0.0004177093505859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0218505859375 mean=-0.000347137451171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.035888671875 mean=9.632110595703125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0240478515625 mean=-0.00090789794921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0361328125 mean=0.0002956390380859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.021240234375 mean=-0.000579833984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03662109375 mean=0.00018024444580078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02392578125 mean=-0.00098419189453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.033935546875 mean=-9.822845458984375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0198974609375 mean=-0.0004863739013671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0341796875 mean=-0.0001316070556640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0218505859375 mean=-0.0011138916015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0380859375 mean=-0.00011301040649414062 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0240478515625 mean=-0.00067901611328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03955078125 mean=1.9788742065429688e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.024169921875 mean=-0.00106048583984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.040283203125 mean=0.00019550323486328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.031005859375 mean=-0.0014190673828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037353515625 mean=9.250640869140625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.053466796875 mean=-0.0024566650390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03857421875 mean=0.0001049041748046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.041259765625 mean=-0.001983642578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.043212890625 mean=-0.00012493133544921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.044189453125 mean=0.002685546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.93s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_880-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 78.38s +Conversion finished in 475.18402004241943 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_880-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_880-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_daemon_880/mcore2hf-config.n9XkHZ ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0000040.done b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0000040.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0000080.done b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0000080.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0000160.done b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0000160.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0000240.done b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0000240.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0000320.done b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0000320.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0000400.done b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0000400.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0000480.done b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0000480.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0000560.done b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0000560.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0000640.done b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0000640.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0000800.done b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0000800.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0000880.done b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0000880.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0000960.done b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0000960.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0001120.done b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0001120.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0001200.done b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0001200.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0001280.done b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0001280.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0001360.done b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0001360.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0001440.done b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0001440.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0001520.done b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0001520.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0001600.done b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0001600.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0001760.done b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0001760.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0001840.done b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0001840.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0001920.done b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0001920.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0002000.done b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0002000.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0002240.done b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0002240.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0002320.done b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0002320.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0002384.done b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/done/0002384.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/hf_export_daemon_stage16a.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/hf_export_daemon_stage16a.log new file mode 100644 index 0000000000000000000000000000000000000000..5919680104683b226573fe45e8e75a304a69eedf --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/hf_export_daemon_stage16a.log @@ -0,0 +1,132 @@ +[2026-10-11 16:57:28] daemon start: ckpt=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints interval=80 s3=s3://yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf +[2026-10-11 18:52:40] iter 80: preserving ckpt (hardlinks)... +[2026-10-11 18:52:40] iter 80: converting to HF (CPU)... +[2026-10-11 19:01:42] iter 80: uploading to s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0000080-hf ... +[2026-10-11 19:13:40] iter 80: exported -> s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0000080-hf (verified 68231323822 bytes), local cleaned +[2026-10-11 23:55:12] iter 160: preserving ckpt (hardlinks)... +[2026-10-11 23:55:12] iter 160: converting to HF (CPU)... +[2026-10-12 00:04:43] iter 160: uploading to s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0000160-hf ... +[2026-10-12 00:15:55] iter 160: exported -> s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0000160-hf (verified 68231323822 bytes), local cleaned +[2026-10-12 04:58:28] iter 240: preserving ckpt (hardlinks)... +[2026-10-12 04:58:28] iter 240: converting to HF (CPU)... +[2026-10-12 05:07:52] iter 240: uploading to s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0000240-hf ... +[2026-10-12 05:18:16] iter 240: exported -> s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0000240-hf (verified 68231323822 bytes), local cleaned +[2026-10-12 10:01:49] iter 320: preserving ckpt (hardlinks)... +[2026-10-12 10:01:50] iter 320: converting to HF (CPU)... +[2026-10-12 10:11:19] iter 320: uploading to s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0000320-hf ... +[2026-10-12 10:21:39] iter 320: exported -> s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0000320-hf (verified 68231323822 bytes), local cleaned +[2026-10-12 15:03:12] iter 400: preserving ckpt (hardlinks)... +[2026-10-12 15:03:12] iter 400: converting to HF (CPU)... +[2026-10-12 15:12:45] iter 400: uploading to s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0000400-hf ... +[2026-10-12 15:22:54] iter 400: exported -> s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0000400-hf (verified 68231323822 bytes), local cleaned +[2026-10-12 20:06:33] iter 480: preserving ckpt (hardlinks)... +[2026-10-12 20:06:33] iter 480: converting to HF (CPU)... +[2026-10-12 20:16:02] iter 480: uploading to s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0000480-hf ... +[2026-10-12 20:26:44] iter 480: exported -> s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0000480-hf (verified 68231323822 bytes), local cleaned +[2026-10-13 01:09:32] iter 560: preserving ckpt (hardlinks)... +[2026-10-13 01:09:33] iter 560: converting to HF (CPU)... +[2026-10-13 01:19:18] iter 560: uploading to s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0000560-hf ... +[2026-10-13 01:29:53] iter 560: exported -> s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0000560-hf (verified 68231323822 bytes), local cleaned +[2026-10-13 06:11:27] iter 640: preserving ckpt (hardlinks)... +[2026-10-13 06:11:28] iter 640: converting to HF (CPU)... +[2026-10-13 06:21:05] iter 640: uploading to s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0000640-hf ... +[2026-10-13 06:31:13] iter 640: exported -> s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0000640-hf (verified 68231323822 bytes), local cleaned +[2026-10-13 11:12:47] iter 720: preserving ckpt (hardlinks)... +[2026-10-13 11:12:47] iter 720: converting to HF (CPU)... +[2026-10-13 11:13:57] iter 720: CONVERSION FAILED (rc=1), keeping preserve for debug +[2026-10-13 11:14:57] iter 720: preserving ckpt (hardlinks)... +[2026-10-13 11:14:58] iter 720: converting to HF (CPU)... +[2026-10-13 11:24:11] iter 720: uploading to s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0000720-hf ... +[2026-10-13 11:35:21] iter 720: exported -> s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0000720-hf (verified 68231323822 bytes), local cleaned +[2026-10-13 16:14:55] iter 800: preserving ckpt (hardlinks)... +[2026-10-13 16:14:55] iter 800: converting to HF (CPU)... +[2026-10-13 16:24:20] iter 800: uploading to s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0000800-hf ... +[2026-10-13 16:35:06] iter 800: exported -> s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0000800-hf (verified 68231323822 bytes), local cleaned +[2026-10-13 21:16:40] iter 880: preserving ckpt (hardlinks)... +[2026-10-13 21:16:40] iter 880: converting to HF (CPU)... +[2026-10-13 21:27:24] iter 880: uploading to s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0000880-hf ... +[2026-10-13 21:38:07] iter 880: exported -> s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0000880-hf (verified 68231323822 bytes), local cleaned +[2026-10-14 02:18:40] iter 960: preserving ckpt (hardlinks)... +[2026-10-14 02:18:40] iter 960: converting to HF (CPU)... +[2026-10-14 02:29:47] iter 960: uploading to s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0000960-hf ... +[2026-10-14 02:39:58] iter 960: exported -> s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0000960-hf (verified 68231323822 bytes), local cleaned +[2026-10-14 07:20:31] iter 1040: preserving ckpt (hardlinks)... +[2026-10-14 07:20:31] iter 1040: converting to HF (CPU)... +[2026-10-14 07:32:07] iter 1040: uploading to s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0001040-hf ... +[2026-10-14 07:42:38] iter 1040: exported -> s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0001040-hf (verified 68231323822 bytes), local cleaned +[2026-10-14 12:23:11] iter 1120: preserving ckpt (hardlinks)... +[2026-10-14 12:23:11] iter 1120: converting to HF (CPU)... +[2026-10-14 12:32:40] iter 1120: uploading to s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0001120-hf ... +[2026-10-14 12:43:23] iter 1120: exported -> s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0001120-hf (verified 68231323822 bytes), local cleaned +[2026-10-14 17:24:58] iter 1200: preserving ckpt (hardlinks)... +[2026-10-14 17:24:58] iter 1200: converting to HF (CPU)... +[2026-10-14 17:34:39] iter 1200: uploading to s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0001200-hf ... +[2026-10-14 17:45:34] iter 1200: exported -> s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0001200-hf (verified 68231323822 bytes), local cleaned +[2026-10-14 22:26:08] iter 1280: preserving ckpt (hardlinks)... +[2026-10-14 22:26:08] iter 1280: converting to HF (CPU)... +[2026-10-14 22:36:31] iter 1280: uploading to s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0001280-hf ... +[2026-10-14 22:47:40] iter 1280: exported -> s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0001280-hf (verified 68231323822 bytes), local cleaned +[2026-10-15 03:28:13] iter 1360: preserving ckpt (hardlinks)... +[2026-10-15 03:28:13] iter 1360: converting to HF (CPU)... +[2026-10-15 03:37:27] iter 1360: uploading to s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0001360-hf ... +[2026-10-15 03:47:53] iter 1360: exported -> s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0001360-hf (verified 68231323822 bytes), local cleaned +[2026-10-15 08:28:25] iter 1440: preserving ckpt (hardlinks)... +[2026-10-15 08:28:25] iter 1440: converting to HF (CPU)... +[2026-10-15 08:37:56] iter 1440: uploading to s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0001440-hf ... +[2026-10-15 08:48:26] iter 1440: exported -> s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0001440-hf (verified 68231323822 bytes), local cleaned +[2026-10-15 13:29:59] iter 1520: preserving ckpt (hardlinks)... +[2026-10-15 13:29:59] iter 1520: converting to HF (CPU)... +[2026-10-15 13:39:05] iter 1520: uploading to s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0001520-hf ... +[2026-10-15 13:49:32] iter 1520: exported -> s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0001520-hf (verified 68231323822 bytes), local cleaned +[2026-10-15 18:32:07] iter 1600: preserving ckpt (hardlinks)... +[2026-10-15 18:32:07] iter 1600: converting to HF (CPU)... +[2026-10-15 18:41:25] iter 1600: uploading to s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0001600-hf ... +[2026-10-15 18:51:35] iter 1600: exported -> s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0001600-hf (verified 68231323822 bytes), local cleaned +[2026-10-15 23:33:08] iter 1680: preserving ckpt (hardlinks)... +[2026-10-15 23:33:08] iter 1680: converting to HF (CPU)... +[2026-10-15 23:42:41] iter 1680: uploading to s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0001680-hf ... +[2026-10-15 23:53:11] iter 1680: exported -> s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0001680-hf (verified 68231323822 bytes), local cleaned +[2026-10-16 04:34:44] iter 1760: preserving ckpt (hardlinks)... +[2026-10-16 04:34:44] iter 1760: converting to HF (CPU)... +[2026-10-16 04:35:50] iter 1760: CONVERSION FAILED (rc=1), keeping preserve for debug +[2026-10-16 04:36:51] iter 1760: preserving ckpt (hardlinks)... +[2026-10-16 04:36:51] iter 1760: converting to HF (CPU)... +[2026-10-16 04:45:40] iter 1760: uploading to s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0001760-hf ... +[2026-10-16 04:56:05] iter 1760: exported -> s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0001760-hf (verified 68231323822 bytes), local cleaned +[2026-10-16 09:36:38] iter 1840: preserving ckpt (hardlinks)... +[2026-10-16 09:36:38] iter 1840: converting to HF (CPU)... +[2026-10-16 09:46:09] iter 1840: uploading to s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0001840-hf ... +[2026-10-16 09:56:44] iter 1840: exported -> s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0001840-hf (verified 68231323822 bytes), local cleaned +[2026-10-16 14:37:18] iter 1920: preserving ckpt (hardlinks)... +[2026-10-16 14:37:18] iter 1920: converting to HF (CPU)... +[2026-10-16 14:46:49] iter 1920: uploading to s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0001920-hf ... +[2026-10-16 14:57:21] iter 1920: exported -> s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0001920-hf (verified 68231323822 bytes), local cleaned +[2026-10-16 19:38:54] iter 2000: preserving ckpt (hardlinks)... +[2026-10-16 19:38:55] iter 2000: converting to HF (CPU)... +[2026-10-16 19:48:40] iter 2000: uploading to s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0002000-hf ... +[2026-10-16 19:59:01] iter 2000: exported -> s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0002000-hf (verified 68231323822 bytes), local cleaned +[2026-10-17 00:39:34] iter 2080: preserving ckpt (hardlinks)... +[2026-10-17 00:39:34] iter 2080: converting to HF (CPU)... +[2026-10-17 00:40:41] iter 2080: CONVERSION FAILED (rc=1), keeping preserve for debug +[2026-10-17 00:41:41] iter 2080: preserving ckpt (hardlinks)... +[2026-10-17 00:41:41] iter 2080: converting to HF (CPU)... +[2026-10-17 00:50:55] iter 2080: uploading to s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0002080-hf ... +[2026-10-17 01:01:15] iter 2080: exported -> s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0002080-hf (verified 68231323822 bytes), local cleaned +[2026-10-17 05:40:48] iter 2160: preserving ckpt (hardlinks)... +[2026-10-17 05:40:48] iter 2160: converting to HF (CPU)... +[2026-10-17 05:50:12] iter 2160: uploading to s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0002160-hf ... +[2026-10-17 06:00:35] iter 2160: exported -> s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0002160-hf (verified 68231323822 bytes), local cleaned +[2026-10-17 10:42:08] iter 2240: preserving ckpt (hardlinks)... +[2026-10-17 10:42:09] iter 2240: converting to HF (CPU)... +[2026-10-17 10:51:33] iter 2240: uploading to s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0002240-hf ... +[2026-10-17 11:02:02] iter 2240: exported -> s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0002240-hf (verified 68231323822 bytes), local cleaned +[2026-10-17 15:42:34] iter 2320: preserving ckpt (hardlinks)... +[2026-10-17 15:42:34] iter 2320: converting to HF (CPU)... +[2026-10-17 15:52:11] iter 2320: uploading to s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0002320-hf ... +[2026-10-17 16:03:06] iter 2320: exported -> s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0002320-hf (verified 68231323822 bytes), local cleaned +[2026-10-17 19:50:33] no training process detected; latest iter=2384 +[2026-10-17 19:50:33] iter 2384: preserving ckpt (hardlinks)... +[2026-10-17 19:50:33] iter 2384: converting to HF (CPU)... +[2026-10-17 19:59:32] iter 2384: uploading to s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0002384-hf ... +[2026-10-17 20:10:34] iter 2384: exported -> s3:yulan/f3-plain-shortconv-adamh-stage16a-cp32-512k-hf/iter_0002384-hf (verified 68231323822 bytes), local cleaned +[2026-10-17 20:10:34] final iter 2384 exported; daemon exit